Commit 5b71881b authored by José Henrique's avatar José Henrique
Browse files

Implement concurrent builds

parent d74bf3ed
......@@ -4,9 +4,12 @@ import time
import sys
import os
import copy
import traceback
from build_env import build_env_vars
from build_utils import start_notifier, start_build
from build_utils import start_write_ts_file
from build_utils import cleanup
from post_build import start_post_build
from post_build_env import init_post_build_env_vars
......@@ -15,10 +18,18 @@ notifier_env_vars["TELEGRAM_TOKEN"] = os.getenv("TELEGRAM_TOKEN")
notifier_env_vars["TELEGRAM_CHAT_ID"] = os.getenv("TELEGRAM_CHAT_ID")
if __name__ == "__main__":
start_write_ts_file(build_env_vars["working_dir"])
start_notifier(notifier_env_vars)
if start_build(build_env_vars):
print("Build success, running post build actions...\n")
start_post_build(init_post_build_env_vars(build_env_vars))
else:
try:
if start_build(build_env_vars):
print("Build success, running post build actions...\n")
start_post_build(init_post_build_env_vars(build_env_vars))
else:
raise Exception("Build failed")
except Exception:
traceback.print_exc()
print("Build failed.\n")
cleanup()
sys.exit(1)
cleanup()
sys.exit(0)
import os
import sys
import socket
import hashlib
from build_utils import determine_build_dir
def default_value(key, def_value):
......@@ -50,12 +52,12 @@ else:
sys.exit("Invalid version")
# Others
build_env_vars["working_dir"] = (
"/mnt/roms" if not os.getenv("working_dir")
else os.getenv("working_dir")) + "/pe_" + build_env_vars["version"]
build_env_vars["ccache_dir"] = (
"/mnt/roms/ccache" if not os.getenv("working_dir")
else os.getenv("working_dir"))
build_env_vars["working_dir"] = determine_build_dir(
("/mnt/roms" if not os.getenv("working_dir") else os.getenv("working_dir"))
+ "/pe_" + build_env_vars["version"])
build_env_vars["ccache_dir"] = build_env_vars["working_dir"] + "_ccache"
build_env_vars["build_container_name"] = "android_build_" + hashlib.md5(
build_env_vars["working_dir"].encode("utf-8")).hexdigest()
if not os.path.exists(build_env_vars["working_dir"]):
os.makedirs(build_env_vars["working_dir"])
......
......@@ -6,8 +6,24 @@ import traceback
import threading
import time
import json
import hashlib
import sys
docker_client = docker.from_env()
stop_ts_thread = False
def kill_container(container_name):
try:
docker_client.containers.get(container_name).remove(force=True)
print("Killing old container: " + container_name)
except Exception:
pass
def pull_image(image_name):
print("Pulling image: " + image_name)
docker_client.images.pull(image_name)
def stream_container_logs_thread(container_name):
......@@ -25,40 +41,69 @@ def stream_container_logs(container_name):
args=(container_name,)).start()
def kill_container(container_name):
try:
docker_client.containers.get(container_name).remove(force=True)
print("Killing old container: " + container_name)
except Exception:
pass
def determine_build_dir(base_dir):
max_concurrent = 2
build_dir = base_dir
max_time = 3 # 3 seconds
for i in range(1, max_concurrent + 1):
if i != 1:
build_dir = base_dir + "_" + str(i)
time.sleep(1)
build_container_name = (
"android_build_" +
hashlib.md5(build_dir.encode("utf-8")).hexdigest()
)
if os.path.exists(build_dir):
ts_file = build_dir + "/.ts"
if os.path.exists(ts_file):
if os.stat(ts_file).st_mtime < time.time() - max_time:
kill_container(build_container_name)
return build_dir
else:
return build_dir
else:
return build_dir
sys.exit("Unable to determine build dir: " + build_dir)
def pull_image(image_name):
print("Pulling image: " + image_name)
docker_client.images.pull(image_name)
def write_ts_file_thread(build_dir):
ts_file = build_dir + "/.ts"
while stop_ts_thread is False:
with open(ts_file, "w") as f:
f.write(str(time.time()))
time.sleep(1)
os.remove(ts_file)
def cleanup():
global stop_ts_thread
stop_ts_thread = True
time.sleep(3)
def start_write_ts_file(build_dir):
threading.Thread(target=write_ts_file_thread,
args=(build_dir,)).start()
def start_notifier(environment_vars):
container_name = "android_build_notifier"
image_name = "pixelexperience/android_build_notifier_ci"
volumes_to_mount = {
"/var/run/docker.sock": {
"bind": "/var/run/docker.sock", "mode": "rw"
}
}
kill_container(container_name)
pull_image(image_name)
print("Starting container: " + container_name)
print("Starting notifier container...")
docker_client.containers.run(image_name,
detach=True,
remove=True,
name=container_name,
volumes=volumes_to_mount,
environment=environment_vars)
def start_build(environment_vars):
container_name = "android_build"
container_name = environment_vars["build_container_name"]
image_name = "pixelexperience/android_build_ci"
volumes_to_mount = {
environment_vars["working_dir"]: {
......@@ -71,10 +116,9 @@ def start_build(environment_vars):
"bind": "/home/android_build/.gitcookies", "mode": "ro"
}
}
kill_container(container_name)
pull_image(image_name)
print("Starting container: " + container_name +
", with environment variables: " + json.dumps(environment_vars))
print("Starting build container, with environment variables: " +
json.dumps(environment_vars))
try:
stream_container_logs(container_name)
time.sleep(1)
......
......@@ -162,9 +162,11 @@ def send_to_release_cp(post_build_env_vars):
global incremental_info
print("Uploading build artifacts to releasecp...")
has_target_files = is_incremental_and_target_files_enabled(post_build_env_vars) and (
post_build_env_vars["production"] or
post_build_env_vars["generate_incremental"])
has_target_files = (
is_incremental_and_target_files_enabled(post_build_env_vars) and
(post_build_env_vars["production"] or
post_build_env_vars["generate_incremental"])
)
has_aosp_recovery = post_build_env_vars["aosp_recovery"]
json_str = json.dumps({
......@@ -204,6 +206,7 @@ def send_to_release_cp(post_build_env_vars):
ftp_password,
ftp_parent_dir,
files_to_upload)
print("Files uploaded!")
def start_post_build(post_build_env_vars):
......
......@@ -17,7 +17,8 @@ def run_command(cmd):
return int(bin(result).replace("0b", "").rjust(16, '0')[:8], 2)
def sign_target_files_apks(unsigned_target_files_path, dest_target_files_path, needs_apex_signing):
def sign_target_files_apks(unsigned_target_files_path, dest_target_files_path,
needs_apex_signing):
signing_keys_dir = "signing_keys/default"
cmd = "python2 build/make/tools/releasetools/sign_target_files_apks"
if needs_apex_signing:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment