Commit 31b6d8e3 authored by José Henrique's avatar José Henrique
Browse files

Implement post build actions

parent a7b6fbf2
......@@ -136,3 +136,6 @@ dmypy.json
# Cython debug symbols
cython_debug/
# Others
post_build/
\ No newline at end of file
......@@ -5,7 +5,8 @@ import sys
import os
from build_env import build_env_vars
from docker_utils import start_notifier, start_build
from build_utils import start_notifier, start_build
from post_build import start_post_build
notifier_env_vars = build_env_vars
notifier_env_vars["TELEGRAM_TOKEN"] = os.getenv("TELEGRAM_TOKEN")
......@@ -14,7 +15,8 @@ notifier_env_vars["TELEGRAM_CHAT_ID"] = os.getenv("TELEGRAM_CHAT_ID")
if __name__ == "__main__":
start_notifier(notifier_env_vars)
if start_build(build_env_vars):
print("Build success.\n")
print("Build success, running post build actions...\n")
start_post_build()
else:
print("Build failed.\n")
sys.exit(1)
......@@ -52,8 +52,8 @@ build_env_vars["wip_branch"] = (build_env_vars["production"] is False
build_env_vars["generate_incremental"] = (
build_env_vars["generate_incremental"] == "true")
# Working dir
build_env_vars["working_dir"] = "/mnt/roms/pe_" + build_env_vars["version"]
# Extras
# Others
build_env_vars["working_dir"] = (
"/mnt/roms" if not os.getenv("working_dir")
else os.getenv("working_dir"))
build_env_vars["runner_name"] = build_env_vars["CI_RUNNER_DESCRIPTION"]
......@@ -69,7 +69,7 @@ def start_build(environment_vars):
container_name = "android_build"
image_name = "pixelexperience/android_build_ci"
volumes_to_mount = {
"/mnt/roms": {
environment_vars["working_dir"]: {
"bind": "/roms", "mode": "rw"
},
"/home/gitlab-runner/.gitcookies": {
......
#!/usr/bin/env python3
import os
import sys
import json
from post_build_env import init_post_build_env_vars
from post_build_utils import *
post_build_env_vars = {}
files_to_upload = []
def sync_tools():
for env_var in post_build_env_vars["tools"]:
if os.getenv("DEBUG") and os.path.exists(env_var["path"]):
return
sync_repo(env_var["url"], env_var["path"], env_var["branch"])
def create_release_zip():
target_files_path = (
post_build_env_vars["target_files_path"]
if post_build_env_vars["production"] else
post_build_env_vars["unsigned_target_files_path"])
if post_build_env_vars["production"]:
print("Signing apks...")
sign_target_files_apks(
post_build_env_vars["unsigned_target_files_path"],
target_files_path)
print("Creating release zip file...")
ota_from_target_files(
target_files_path,
post_build_env_vars["build_name"],
sign=post_build_env_vars["production"])
files_to_upload.append(
post_build_env_vars["build_name"])
if (post_build_env_vars["production"]
or post_build_env_vars["generate_incremental"]):
print("Compressing target files using zstd...")
compress_target_files(
target_files_path,
post_build_env_vars["target_files_compressed_path"])
files_to_upload.append(
post_build_env_vars["target_files_compressed_path"])
def extract_recovery():
if post_build_env_vars["aosp_recovery"]:
print("Extracting AOSP recovery...")
target_files_path = (
post_build_env_vars["target_files_path"]
if post_build_env_vars["production"] else
post_build_env_vars["unsigned_target_files_path"])
file_to_extract = "IMAGES/recovery.img"
if (post_build_env_vars["ab_update"] and
exists_on_target_files(target_files_path,
file_to_extract) is False):
file_to_extract = "IMAGES/boot.img"
if not extract_from_target_files(target_files_path, file_to_extract):
sys.exit("Failed to extract recovery")
os.rename(file_to_extract, post_build_env_vars["aosp_recovery_path"])
os.rmdir("IMAGES")
print("AOSP recovery extracted to " +
post_build_env_vars["aosp_recovery_path"])
files_to_upload.append(
post_build_env_vars["aosp_recovery_path"])
def send_to_release_cp():
print("Uploading build artifacts to releasecp...")
has_target_files = (post_build_env_vars["production"] or
post_build_env_vars["generate_incremental"])
has_aosp_recovery = post_build_env_vars["aosp_recovery"]
json_str = json.dumps({
"device": post_build_env_vars["device"],
"version": post_build_env_vars["version"],
"name": post_build_env_vars["build_name"],
"size": os.path.getsize(post_build_env_vars["build_name"]),
"md5_hash": get_md5(post_build_env_vars["build_name"]),
"ab_update": post_build_env_vars["ab_update"],
"has_aosp_recovery": has_aosp_recovery,
"aosp_recovery_md5_hash": (
get_md5(post_build_env_vars["aosp_recovery_path"])
if has_aosp_recovery else
None),
"has_target_files": has_target_files,
"target_files_md5_hash": (
get_md5(post_build_env_vars["target_files_compressed_path"])
if has_target_files else
None)
})
json_path = post_build_env_vars["build_name"].replace(".zip", ".json")
with open(json_path, "w") as file:
file.write(json_str)
print(json_str)
files_to_upload.append(json_path)
ftp_host = os.getenv("FTP_HOST")
ftp_username = os.getenv("FTP_USERNAME")
ftp_password = os.getenv("FTP_PASSWORD")
ftp_parent_dir = post_build_env_vars["device"] + \
"/" + post_build_env_vars["version"]
upload_to_ftp(ftp_host,
ftp_username,
ftp_password,
ftp_parent_dir,
files_to_upload)
def start_post_build():
global post_build_env_vars
post_build_env_vars = init_post_build_env_vars()
sync_tools()
create_release_zip()
extract_recovery()
send_to_release_cp()
if os.getenv("DEBUG"):
start_post_build()
#!/usr/bin/env python3
import os
import glob
import sys
import shutil
from build_env import build_env_vars
def find_file(path):
for file in glob.glob(path):
return file
return None
def parse_build_prop_file(build_prop_path):
props = {}
with open(build_prop_path) as f:
for line in f:
if "=" in line and line.startswith("#") is False:
line = line.split("=", 1)
props[line[0].strip()] = line[1].strip()
return props
def init_post_build_env_vars():
result = build_env_vars
if not os.getenv("DEBUG") and os.path.exists("post_build"):
shutil.rmtree('post_build')
if not os.path.exists("post_build"):
os.mkdir("post_build")
os.chdir("post_build")
working_dir = result["working_dir"] + \
"/pe_" + result["version"]
unsigned_target_files_path_glob = working_dir + \
"/out/target/product/*/obj/PACKAGING/target_files_intermediates/*-target_files-*.zip"
result["unsigned_target_files_path"] = None
build_prop = parse_build_prop_file(
find_file(working_dir +
"/out/target/product/*/system/build.prop"))
default_prop = parse_build_prop_file(
find_file(working_dir +
"/out/target/product/*/system/etc/prop.default"))
if "org.pixelexperience.version.display" not in build_prop:
sys.exit(
"Unable to find org.pixelexperience.version.display in build.prop")
result["build_name"] = (
build_prop["org.pixelexperience.version.display"] + ".zip")
result["ab_update"] = (
"ro.build.ab_update" in build_prop and
build_prop["ro.build.ab_update"] == "true")
result["aosp_recovery"] = result["ab_update"] or (
"persist.sys.recovery_update" in default_prop and
default_prop["persist.sys.recovery_update"] == "true")
result["aosp_recovery_path"] = (
result["build_name"].replace(".zip", ".img"))
result["changelog_path"] = (
find_file(working_dir +
"/out/target/product/*/" +
result["build_name"].replace(".zip", "") + "*.txt"))
if result["changelog_path"] is None:
sys.exit("Unable to find changelog file")
result["unsigned_target_files_path"] = find_file(
unsigned_target_files_path_glob)
if result["unsigned_target_files_path"] is None:
sys.exit("Unable to find file: " + unsigned_target_files_path_glob)
result["target_files_path"] = (
result["build_name"].replace(".zip", "_target_files.zip"))
result["target_files_compressed_path"] = (
result["target_files_path"] + ".zst")
# Repos
build_repo_url = ("https://github.com/PixelExperience/build"
if not os.getenv("build_repo_url") else
os.getenv("build_repo_url"))
result["tools"] = [
{
"url": build_repo_url,
"branch": result["manifest_branch"],
"path": "build/make"
},
{
"url": "https://github.com/PixelExperience/art",
"branch": result["fallback_branch"],
"path": "art"
},
{
"url": "https://github.com/PixelExperience/external_avb",
"branch": result["fallback_branch"],
"path": "external/avb"
},
{
"url": "git@gitlab.pixelexperience.org:infra/tools/signing_keys.git",
"branch": "master",
"path": "signing_keys"
},
{
"url": "git@gitlab.pixelexperience.org:infra/tools/otatools.git",
"branch": result["fallback_branch"],
"path": "out/host/linux-x86"
}
]
return result
#!/usr/bin/env python3
import os
import sys
import git
import traceback
import hashlib
import ftplib
from zipfile import ZipFile
def run_command(cmd):
result = os.system(cmd)
return int(bin(result).replace("0b", "").rjust(16, '0')[:8], 2)
def sign_target_files_apks(unsigned_target_files_path, dest_target_files_path):
signing_keys_dir = "signing_keys/default"
cmd = "python2 build/make/tools/releasetools/sign_target_files_apks"
cmd += " -o -d " + signing_keys_dir
cmd += " " + unsigned_target_files_path
cmd += " " + dest_target_files_path
result = run_command(cmd)
if result != 0:
sys.exit(
"Failed to run sign_target_files_apks, process returned code " +
str(result))
def ota_from_target_files(target_files_path, dest_file_path,
sign=False):
release_key_path = "signing_keys/default/releasekey"
cmd = "python2 build/make/tools/releasetools/ota_from_target_files"
if sign:
cmd += " -k" + release_key_path
cmd += " " + target_files_path
cmd += " " + dest_file_path
result = run_command(cmd)
if result != 0:
sys.exit(
"Failed to run ota_from_target_files, process returned code " +
str(result))
def compress_target_files(target_files_path, dest_file_path):
cmd = "zstd -3 " + target_files_path + " -o " + dest_file_path
result = run_command(cmd)
if result != 0:
sys.exit(
"Failed to run zstd, process returned code " +
str(result))
def sync_repo(url, path, branch):
print("Syncing repo " + url + " (" + branch + " branch) to path " + path)
git.Repo.clone_from(url, path, branch=branch, multi_options=["--depth=1"])
def extract_from_target_files(zip_path, file):
try:
with ZipFile(zip_path) as zip_file:
zip_file.extract(file)
return True
except KeyError:
return False
def exists_on_target_files(zip_path, file):
with ZipFile(zip_path) as zip_file:
return file in zip_file.namelist()
def get_md5(path):
with open(path, "rb") as f:
file_hash = hashlib.md5()
while chunk := f.read(8192):
file_hash.update(chunk)
return file_hash.hexdigest()
def upload_to_ftp(host, username, password, parent_dir, files):
print("Connecting to " + host)
with ftplib.FTP(host) as ftp:
ftp.login(username, password)
for d in parent_dir.split("/"):
if d not in ftp.nlst():
ftp.mkd(d)
ftp.cwd(d)
for file_name in files:
with open(file_name, "rb") as file:
print("Uploading " + file_name)
ftp.storbinary("STOR " + file_name, file)
docker
\ No newline at end of file
docker
GitPython>=3.1.8
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment