Commit 376c9fde authored by shreejoy's avatar shreejoy
Browse files

New validator

parent d8f5a616
target/
\ No newline at end of file
*.pyc
......@@ -17,7 +17,7 @@ build:
git -C official_devices checkout FETCH_HEAD
fi
docker build --no-cache -t official_devices_ci:latest ./
docker run -e RUBYOPT='-W0' -e TELEGRAM_TOKEN=${TELEGRAM_TOKEN} -e GH_PERSONAL_TOKEN=${GH_PERSONAL_TOKEN} -e GIT_PR_NUMBER=${GIT_PR_NUMBER} -e BUILD_URL=${CI_JOB_URL} official_devices_ci:latest
docker run -e TELEGRAM_TOKEN=${TELEGRAM_TOKEN} -e GH_PERSONAL_TOKEN=${GH_PERSONAL_TOKEN} -e GIT_PR_NUMBER=${GIT_PR_NUMBER} -e BUILD_URL=${CI_JOB_URL} official_devices_ci:latest
only:
- triggers
- web
......
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "json"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd"
[[package]]
name = "pixel-ci"
version = "0.1.0"
dependencies = [
"json",
]
[package]
name = "pixel-ci"
version = "0.1.0"
authors = ["baalajimaestro <baalajimaestro@pixelexperience.org>"]
edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[[bin]]
name="json-linter"
path = "lint_json.rs"
[dependencies]
json = "0.12.1"
......@@ -671,4 +671,4 @@ into proprietary programs. If your program is a subroutine library, you
may consider it more useful to permit linking proprietary applications with
the library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License. But first, please read
<http://www.gnu.org/philosophy/why-not-lgpl.html>.
\ No newline at end of file
<http://www.gnu.org/philosophy/why-not-lgpl.html>.
//
// Copyright © 2020 PixelExperience Project
//
// SPDX-License-Identifier: GPL-3.0
//
use json;
use std::fs;
fn test_integrity_devices(json_iter: json::JsonValue) -> String {
let mut return_value = String::new();
for json_value in 0..json_iter.len() {
if json_iter[json_value]["name"].is_empty() {
return_value.push_str("I cant find the device name!");
}
if json_iter[json_value]["brand"].is_empty() {
return_value.push_str("I cant find device brand!");
}
if json_iter[json_value]["codename"].is_empty() {
return_value.push_str("I cant find device codename!");
}
let versions = Clone::clone(&json_iter[json_value]["supported_versions"]);
for version in 0..versions.len() {
if !(&versions[version]["version_code"] == "eleven")
&& !(&versions[version]["version_code"] == "ten")
&& !(&versions[version]["version_code"] == "ten_plus")
&& !(&versions[version]["version_code"] == "pie")
&& !(&versions[version]["version_code"] == "pie_plus")
{
return_value.push_str("Unsupported Version has been pushed!");
}
}
}
return return_value;
}
fn test_integrity_team(json_iter: json::JsonValue) -> String {
let mut return_value = String::new();
// Sometimes the file can contain just one entry
if !json_iter["name"].is_empty() {
if json_iter["country"].is_empty() {
return_value.push_str("I cant find maintainer country!");
}
if json_iter["github_username"].is_empty() {
return_value.push_str("I cant find maintainer github username!");
}
let devices = json_iter["devices"].clone();
for device in 0..devices.len() {
if devices[device]["codename"].is_empty() {
return_value.push_str("I cant find maintainer github username!");
}
let versions = devices[device]["versions"].clone();
for version in 0..versions.len() {
if !(&versions[version] == "eleven")
&& !(&versions[version] == "ten")
&& !(&versions[version] == "ten_plus")
&& !(&versions[version] == "pie")
&& !(&versions[version] == "pie_plus")
{
return_value.push_str("Unsupported Version has been pushed!");
}
}
}
} else {
for json_value in 0..json_iter.len() {
if json_iter[json_value]["country"].is_empty() {
return_value.push_str("I cant find maintainer country!");
}
if json_iter[json_value]["github_username"].is_empty() {
return_value.push_str("I cant find maintainer github username!");
}
let devices = json_iter[json_value]["devices"].clone();
for device in 0..devices.len() {
if devices[device]["codename"].is_empty() {
return_value.push_str("I cant find maintainer github username!");
}
let versions = devices[device]["versions"].clone();
for version in 0..versions.len() {
if !(&versions[version] == "eleven")
&& !(&versions[version] == "ten")
&& !(&versions[version] == "ten_plus")
&& !(&versions[version] == "pie")
&& !(&versions[version] == "pie_plus")
{
return_value.push_str("Unsupported Version has been pushed!");
}
}
}
}
}
return return_value;
}
fn main() {
let data = fs::read_to_string("official_devices/devices.json").expect("Unable to read file");
println!("Running Integrity Tests on official_devices/devices.json.....");
fs::write("/tmp/current_file", "devices.json").expect("Unable to write file");
let parsed = json::parse(&data).unwrap_or_else(|error| {
println!("\tError parsing JSON: {}", error);
fs::write("/tmp/error_message", error.to_string()).expect("Unable to write file");
panic!("Invalid JSON");
});
let resp = test_integrity_devices(parsed.clone());
if !resp.is_empty() {
println!("Integrity Tests Failed!");
fs::write("/tmp/error_message", resp.to_string()).expect("Unable to write file");
panic!("{}", resp);
} else {
let stringifed = json::stringify_pretty(parsed, 3);
fs::write("official_devices/devices.json", stringifed).expect("Unable to write file");
}
let entries = fs::read_dir("official_devices/team/").unwrap();
for entry in entries {
let path = entry.unwrap().path();
println!("Running Integrity Tests on {:?}", path);
let current_entry = path.clone();
let current_file = current_entry
.clone()
.into_os_string()
.into_string()
.unwrap();
let data2 = fs::read_to_string(path).expect("Unable to read file");
fs::write("/tmp/current_file", current_file).expect("Unable to write file");
let parsed_ = json::parse(&data2).unwrap_or_else(|error| {
println!("\tError parsing JSON: {}", error);
fs::write("/tmp/error_message", error.to_string()).expect("Unable to write file");
panic!("Invalid JSON");
});
let resp2 = test_integrity_team(parsed_.clone());
if !resp2.is_empty() {
println!("Integrity Tests Failed!");
fs::write("/tmp/error_message", resp.to_string()).expect("Unable to write file");
panic!("{}", resp2);
} else {
let stringifed = json::stringify_pretty(parsed_, 3);
fs::write(current_entry, stringifed).expect("Unable to write file");
}
}
fs::remove_file("/tmp/current_file").unwrap();
println!("Success!");
}
......@@ -33,28 +33,24 @@ function approvePR() {
curl -s -X POST -d "{\"commit_id\": \"${COMMIT_HASH}\", \"event\": \"APPROVE\"}" -H "Authorization: token $GH_PERSONAL_TOKEN" "https://api.github.com/repos/PixelExperience/official_devices/pulls/$GIT_PR_NUMBER/reviews" > /dev/null
}
function checkLint() {
function CheckCommit() {
if [[ "$COMMIT_MESSAGE" =~ "[PIXEL-CI]" ]]; then
if [[ -n "$GIT_PR_NUMBER" ]]; then
commentPR "Please remove \"[PIXEL-CI]\" from your commit message."
exit 0
else
echo "Commit already linted, nothing to do."
echo "Commit already formatted, nothing to do."
exit 0
fi
fi
}
function checkJsons() {
cargo run --release
function Validator() {
python3 validator.py
RESULT=$?
if [ "$RESULT" -ne 0 ]; then
ERROR_MESSAGE="$(cat /tmp/error_message)"
FAILED_FILE="$(cat /tmp/current_file)"
fi
if [ -n "$GIT_PR_NUMBER" ]; then
if [ "$RESULT" -ne 0 ]; then
message="JSON validation failed\nFile: ${FAILED_FILE}\nError message: ${ERROR_MESSAGE}"
message="JSON validation failed. Please check ${BUILD_URL} to find possible error."
commentPR "$message"
echo -e "$message"
else
......@@ -63,7 +59,7 @@ function checkJsons() {
echo -e "$message"
fi
elif [ "$RESULT" -ne 0 ]; then
sendTG "Someone has merged a failing file. Please look in ASAP.\nError message: ${ERROR_MESSAGE}\nFile: ${FAILED_FILE}\n${ADMINS}"
sendTG "Someone has merged a failing file. Please check console to find possible error.\n\n${ADMINS}"
else
pushToGit
fi
......@@ -71,7 +67,6 @@ function checkJsons() {
function pushToGit() {
if [ -z "$GIT_PR_NUMBER" ] && [ -n "$(${GIT_CMD} status -s)" ]; then
${GIT_CMD} pull --no-edit origin master
${GIT_CMD} add --all
${GIT_CMD} commit -m "[PIXEL-CI]: Lint files"
${GIT_CMD} remote set-url origin "https://pixelexperiencebot:${GH_PERSONAL_TOKEN}@github.com/PixelExperience/official_devices.git"
......@@ -90,5 +85,5 @@ function pushToGit() {
fi
}
checkLint
checkJsons
checkCommit
Validator
import operator
import json
import sys
import os.path
import requests
SUGGESTIONS = []
ALLOWED_VERSIONS = ['eleven', 'ten', 'ten_plus', 'pie', 'pie_plus']
WIKI_BASE_URL = 'https://raw.githubusercontent.com/PixelExperience/wiki/master/_data/devices'
def test_integrity_devices():
try:
devices = json.loads(open('devices.json', 'r').read())
except:
SUGGESTIONS.append('devices.json is not a valid JSON.')
return
for device in devices:
device_index = devices.index(device)
if 'name' not in device:
SUGGESTIONS.append(
f"'name' key not found in devices.json for device index value {device_index}")
if 'brand' not in device:
SUGGESTIONS.append(
f"'brand' key not found in devices.json for device index value {device_index}")
if 'codename' not in device:
SUGGESTIONS.append(
f"'codename' key not found in devices.json for device index value {device_index}")
else:
codename = device.get('codename')
if not os.path.isfile(f"images/{codename}.png"):
SUGGESTIONS.append(
f"images/{codename}.png not found for codename {codename} in devices.json for device index value {device_index}")
yaml = requests.head(f"{WIKI_BASE_URL}/{codename}.yml")
if yaml.status_code != 200:
SUGGESTIONS.append(
f"Wiki YAML not found for codename {codename} in devices.json for device index value {device_index}")
if 'supported_versions' not in device:
SUGGESTIONS.append(
f"'supported_versions' key not found in devices.json for index value {device_index}")
continue
supported_versions = device['supported_versions']
for supported_version in supported_versions:
version_index = supported_versions.index(supported_version)
if 'version_code' not in supported_version:
SUGGESTIONS.append(
f"'version_code' key not found in devices.json for device index value {device_index} and version index value {version_index}")
if supported_version['version_code'] not in ALLOWED_VERSIONS:
SUGGESTIONS.append(
f"'version_code' key has a value that is not allowed in devices.json for device index value {device_index} and version index value {version_index}")
return
def test_integrity_team():
try:
lead = json.loads(open('team/lead.json', 'r').read())
except:
SUGGESTIONS.append('team/lead.json is not a valid JSON.')
return
try:
core = json.loads(open('team/core.json', 'r').read())
except:
SUGGESTIONS.append('team/core.json is not a valid JSON.')
return
try:
maintainers = json.loads(open('team/maintainers.json', 'r').read())
except:
SUGGESTIONS.append('team/maintainers.json is not a valid JSON.')
return
for member in maintainers:
member_index = maintainers.index(member)
if 'name' not in member:
SUGGESTIONS.append(
f"'name' key not found in maintainers.json for member index value {member_index}")
if 'country' not in member:
SUGGESTIONS.append(
f"'country' key not found in maintainers.json for member index value {member_index}")
if 'github_username' not in member:
SUGGESTIONS.append(
f"'github_username' key not found in maintainers.json for member index value {member_index}")
if 'devices' not in member:
SUGGESTIONS.append(
f"'devices' key not found in maintainers.json for member index value {member_index}")
continue
devices = member['devices']
for device in devices:
device_index = devices.index(device)
if 'codename' not in device:
SUGGESTIONS.append(
f"'codename' key not found in maintainers.json for member index value {member_index} and device index value {device_index}")
if 'versions' not in device:
SUGGESTIONS.append(
f"'versions' key not found in maintainers.json for member index value {member_index}")
continue
versions = device['versions']
for version in versions:
version_index = versions.index(version)
if version not in ALLOWED_VERSIONS:
SUGGESTIONS.append(
f"'versions' key has a value that is not allowed in maintainers.json for device index value {device_index} and version index value {version_index} and member index value {member_index}")
devices = lead['devices']
for device in devices:
device_index = devices.index(device)
if 'codename' not in device:
SUGGESTIONS.append(
f"'codename' key not found in lead.json for device index value {device_index}")
if 'versions' not in device:
SUGGESTIONS.append(
f"'versions' key not found in lead.json for device index value {device_index}")
continue
versions = device['versions']
for version in versions:
version_index = versions.index(version)
if version not in ALLOWED_VERSIONS:
SUGGESTIONS.append(
f"'versions' key has a value that is not allowed in lead.json for device index value {device_index} and version index value {version_index}")
if 'name' not in lead:
SUGGESTIONS.append(
f"'name' key not found in lead.json")
if 'country' not in lead:
SUGGESTIONS.append(
f"'country' key not found in lead.json")
if 'github_username' not in lead:
SUGGESTIONS.append(
f"'github_username' key not found in lead.json")
if 'devices' not in lead:
SUGGESTIONS.append(
f"'devices' key not found in lead.json")
for member in core:
member_index = core.index(member)
if 'name' not in member:
SUGGESTIONS.append(
f"'name' key not found in core.json for member index value {member_index}")
if 'country' not in member:
SUGGESTIONS.append(
f"'country' key not found in core.json for member index value {member_index}")
if 'github_username' not in member:
SUGGESTIONS.append(
f"'github_username' key not found in core.json for member index value {member_index}")
if 'devices' not in member:
SUGGESTIONS.append(
f"'devices' key not found in core.json for member index value {member_index}")
continue
devices = member['devices']
for device in devices:
device_index = devices.index(device)
if 'codename' not in device:
SUGGESTIONS.append(
f"'codename' key not found in core.json for member index value {member_index} and device index value {device_index}")
if 'versions' not in device:
SUGGESTIONS.append(
f"'versions' key not found in core.json for member index value {member_index}")
continue
versions = device['versions']
for version in versions:
version_index = versions.index(version)
if version not in ALLOWED_VERSIONS:
SUGGESTIONS.append(
f"'versions' key has a value that is not allowed in core.json for device index value {device_index} and version index value {version_index} and member index value {member_index}")
return
def format_json():
# Load JSON's
devices = json.loads(open('devices.json', 'r').read())
lead = json.loads(open('team/lead.json', 'r').read())
core = json.loads(open('team/core.json', 'r').read())
maintainers = json.loads(open('team/maintainers.json', 'r').read())
# Sort JSON alphabatically with selected keys
devices = sorted(devices, key=operator.itemgetter('codename', 'brand'))
#lead = sorted(lead, key=operator.itemgetter('name', 'country'))
core = sorted(core, key=operator.itemgetter('name', 'country'))
maintainers = sorted(
maintainers, key=operator.itemgetter('name', 'country'))
# Dump them
devices = json.dumps(devices, indent=3, sort_keys=False)
lead = json.dumps(lead, indent=3, sort_keys=False)
core = json.dumps(core, indent=3, sort_keys=False)
maintainers = json.dumps(maintainers, indent=3, sort_keys=False)
# Open JSON's
devices_json = open('devices.json', 'w')
lead_json = open('team/lead.json', 'w')
core_json = open('team/core.json', 'w')
maintainers_json = open('team/maintainers.json', 'w')
# Write dump's
devices_json.write(devices + '\n')
lead_json.write(lead + '\n')
core_json.write(core + '\n')
maintainers_json.write(maintainers + '\n')
# Close JSON's
devices_json.close()
lead_json.close()
core_json.close()
maintainers_json.close()
return 0
def main():
print('Running Integrity tests for all the JSON\'s.')
test_devices = test_integrity_devices()
test_team = test_integrity_team()
if len(SUGGESTIONS) > 0:
print('Integrity test for one or more JSON\'s failed. Cannot proceed with JSON formatter.\n\n')
print('Below might be the reasons for test failure.')
for SUGGESTION in SUGGESTIONS:
print(SUGGESTION)
sys.exit(1)
else:
print('Integrity test passed successfully. Now formatting all the JSON\'s')
format_json()
sys.exit(0)
if __name__ == "__main__":
main()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment