Commit 619bdf9e authored by José Henrique's avatar José Henrique
Browse files

Fixes

parent dc0e2d4c
......@@ -8,7 +8,7 @@ import traceback
from build_env import build_env_vars
from build_utils import start_notifier, start_build
from build_utils import start_write_ts_file
from build_utils import write_build_data
from build_utils import cleanup
from post_build import start_post_build
from post_build_env import init_post_build_env_vars
......@@ -18,7 +18,7 @@ notifier_env_vars["TELEGRAM_TOKEN"] = os.getenv("TELEGRAM_TOKEN")
notifier_env_vars["TELEGRAM_CHAT_ID"] = os.getenv("TELEGRAM_CHAT_ID")
if __name__ == "__main__":
start_write_ts_file(build_env_vars["working_dir"])
write_build_data(build_env_vars)
start_notifier(notifier_env_vars)
try:
if start_build(build_env_vars):
......@@ -28,6 +28,6 @@ if __name__ == "__main__":
raise Exception("Build failed.")
except Exception:
traceback.print_exc()
cleanup()
cleanup(build_env_vars)
sys.exit(1)
cleanup()
cleanup(build_env_vars)
import os
import sys
import socket
import hashlib
from build_utils import determine_build_dir
......@@ -50,12 +49,22 @@ else:
sys.exit("Invalid version")
# Others
build_env_vars["working_dir"] = determine_build_dir(
build_env_vars["build_id"] = (
build_env_vars["device"] + "_" + build_env_vars["version"]
)
build_env_vars["build_data_dir"] = "/mnt/roms/data"
build_env_vars["build_data_file"] = (
build_env_vars["build_data_dir"] + "/" +
build_env_vars["build_id"] + ".json"
)
build_env_vars["base_working_dir"] = (
("/mnt/roms" if not os.getenv("working_dir") else os.getenv("working_dir"))
+ "/pe_" + build_env_vars["version"])
+ "/pe_" + build_env_vars["version"]
)
build_env_vars["working_dir"] = determine_build_dir(
build_env_vars["build_data_dir"], build_env_vars["base_working_dir"]
)
build_env_vars["ccache_dir"] = build_env_vars["working_dir"] + "_ccache"
build_env_vars["build_container_name"] = "android_build_" + hashlib.md5(
build_env_vars["working_dir"].encode("utf-8")).hexdigest()
if not os.path.exists(build_env_vars["working_dir"]):
os.makedirs(build_env_vars["working_dir"])
......
......@@ -6,11 +6,10 @@ import traceback
import threading
import time
import json
import hashlib
import sys
import glob
docker_client = docker.from_env()
stop_ts_thread = False
def kill_container(container_name):
......@@ -21,6 +20,28 @@ def kill_container(container_name):
pass
def kill_process(pid):
try:
os.kill(pid)
except OSError:
pass
def is_process_running(pid):
try:
os.kill(pid, 0)
except OSError:
return False
else:
return True
def load_json(path):
with open(path) as file_contents:
return json.load(file_contents)
return False
def pull_image(image_name):
print("Pulling image: " + image_name)
docker_client.images.pull(image_name)
......@@ -41,52 +62,45 @@ def stream_container_logs(container_name):
args=(container_name,)).start()
def determine_build_dir(base_dir):
max_concurrent = 2
def determine_build_dir(build_data_dir, base_dir):
if not os.path.exists(build_data_dir):
os.mkdir(build_data_dir)
build_dir = base_dir
max_time = 3 # 3 seconds
max_concurrent = 2
for i in range(1, max_concurrent + 1):
if i != 1:
build_dir = base_dir + "_" + str(i)
build_container_name = (
"android_build_" +
hashlib.md5(build_dir.encode("utf-8")).hexdigest()
)
if os.path.exists(build_dir):
ts_file = build_dir + "/.ts"
if os.path.exists(ts_file):
current_time = int(time.time())
ts_file_m_time = int(os.stat(ts_file).st_mtime)
if current_time - ts_file_m_time >= max_time:
kill_container(build_container_name)
return build_dir
else:
print(ts_file + " modification time: " +
str(ts_file_m_time))
print("current time: " + str(current_time))
else:
return build_dir
else:
is_working_dir_busy = False
for json_file in glob.glob(build_data_dir + '/*.json'):
json_data = load_json(json_file)
if not is_process_running(json_data["pid"]):
os.remove(json_file)
elif json_data["working_dir"] == build_dir:
is_working_dir_busy = True
if not is_working_dir_busy:
return build_dir
sys.exit("Unable to determine build dir: " + build_dir)
def write_ts_file_thread(build_dir):
ts_file = build_dir + "/.ts"
while stop_ts_thread is False:
with open(ts_file, "w") as f:
f.write(str(time.time()))
time.sleep(1)
def cleanup():
global stop_ts_thread
stop_ts_thread = True
time.sleep(3)
def start_write_ts_file(build_dir):
threading.Thread(target=write_ts_file_thread,
args=(build_dir,)).start()
def write_build_data(build_env):
build_id = build_env["build_id"]
working_dir = build_env["working_dir"]
build_data_dir = build_env["build_data_dir"]
build_data_file = build_env["build_data_file"]
if not os.path.exists(build_data_dir):
os.mkdir(path)
if os.path.exists(build_data_file):
json_data = load_json(build_data_file)
kill_process(json_data["pid"])
kill_container(json_data["build_id"])
os.remove(build_data_file)
data = {
"build_id": build_id,
"pid": os.getpid(),
"working_dir": working_dir
}
with open(build_data_file, 'w') as json_file:
json.dump(data, json_file)
def start_notifier(environment_vars):
......@@ -94,19 +108,32 @@ def start_notifier(environment_vars):
volumes_to_mount = {
"/var/run/docker.sock": {
"bind": "/var/run/docker.sock", "mode": "rw"
},
environment_vars["build_data_dir"]: {
"bind": "/mnt/roms/data", "mode": "rw"
}
}
pull_image(image_name)
print("Starting notifier container...")
container_name = "notifier_" + environment_vars["build_id"]
docker_client.containers.run(image_name,
detach=True,
remove=True,
name=container_name,
volumes=volumes_to_mount,
environment=environment_vars)
def cleanup(build_env):
build_data_file = build_env["build_data_file"]
if os.path.exists(build_data_file):
json_data = load_json(build_data_file)
kill_container(json_data["build_id"])
os.remove(build_data_file)
def start_build(environment_vars):
container_name = environment_vars["build_container_name"]
container_name = environment_vars["build_id"]
image_name = "pixelexperience/android_build_ci"
volumes_to_mount = {
environment_vars["working_dir"]: {
......
......@@ -125,7 +125,7 @@ def validate_release_tools(release_tools_path):
import_name = match.group(groupNum)
if import_name not in allowed_imports:
raise Exception("releasetools: Import not allowed: " +
import_name)
import_name)
text = text.replace(" ", "")
for keyword in prohibited_keywords:
if keyword in text:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment