New validator

.gitignore

-target/
\ No newline at end of file
+*.pyc

.gitlab-ci.yml

@@ -17,7 +17,7 @@ build:
         git -C official_devices checkout FETCH_HEAD
     fi
     docker build --no-cache -t official_devices_ci:latest ./
-    docker run -e RUBYOPT='-W0' -e TELEGRAM_TOKEN=${TELEGRAM_TOKEN} -e GH_PERSONAL_TOKEN=${GH_PERSONAL_TOKEN} -e GIT_PR_NUMBER=${GIT_PR_NUMBER} -e BUILD_URL=${CI_JOB_URL} official_devices_ci:latest
+    docker run -e TELEGRAM_TOKEN=${TELEGRAM_TOKEN} -e GH_PERSONAL_TOKEN=${GH_PERSONAL_TOKEN} -e GIT_PR_NUMBER=${GIT_PR_NUMBER} -e BUILD_URL=${CI_JOB_URL} official_devices_ci:latest
   only:
     - triggers
     - web

Cargo.lock

-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-[[package]]
-name = "json"
-version = "0.12.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd"
-
-[[package]]
-name = "pixel-ci"
-version = "0.1.0"
-dependencies = [
- "json",
-]

Cargo.toml

-[package]
-name = "pixel-ci"
-version = "0.1.0"
-authors = ["baalajimaestro <baalajimaestro@pixelexperience.org>"]
-edition = "2018"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[[bin]]
-name="json-linter"
-path = "lint_json.rs"
-
-[dependencies]
-json = "0.12.1"
-

LICENSE

@@ -671,4 +671,4 @@ into proprietary programs.  If your program is a subroutine library, you
 may consider it more useful to permit linking proprietary applications with
 the library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.  But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
\ No newline at end of file
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.

lint_json.rs

-//
-// Copyright © 2020 PixelExperience Project
-//
-// SPDX-License-Identifier: GPL-3.0
-//
-
-use json;
-use std::fs;
-
-fn test_integrity_devices(json_iter: json::JsonValue) -> String {
-    let mut return_value = String::new();
-
-    for json_value in 0..json_iter.len() {
-        if json_iter[json_value]["name"].is_empty() {
-            return_value.push_str("I cant find the device name!");
-        }
-
-        if json_iter[json_value]["brand"].is_empty() {
-            return_value.push_str("I cant find device brand!");
-        }
-
-        if json_iter[json_value]["codename"].is_empty() {
-            return_value.push_str("I cant find device codename!");
-        }
-
-        let versions = Clone::clone(&json_iter[json_value]["supported_versions"]);
-
-        for version in 0..versions.len() {
-            if !(&versions[version]["version_code"] == "eleven")
-                && !(&versions[version]["version_code"] == "ten")
-                && !(&versions[version]["version_code"] == "ten_plus")
-                && !(&versions[version]["version_code"] == "pie")
-                && !(&versions[version]["version_code"] == "pie_plus")
-            {
-                return_value.push_str("Unsupported Version has been pushed!");
-            }
-        }
-    }
-    return return_value;
-}
-
-fn test_integrity_team(json_iter: json::JsonValue) -> String {
-    let mut return_value = String::new();
-
-    // Sometimes the file can contain just one entry
-    if !json_iter["name"].is_empty() {
-        if json_iter["country"].is_empty() {
-            return_value.push_str("I cant find maintainer country!");
-        }
-
-        if json_iter["github_username"].is_empty() {
-            return_value.push_str("I cant find maintainer github username!");
-        }
-
-        let devices = json_iter["devices"].clone();
-
-        for device in 0..devices.len() {
-            if devices[device]["codename"].is_empty() {
-                return_value.push_str("I cant find maintainer github username!");
-            }
-
-            let versions = devices[device]["versions"].clone();
-
-            for version in 0..versions.len() {
-                if !(&versions[version] == "eleven")
-                    && !(&versions[version] == "ten")
-                    && !(&versions[version] == "ten_plus")
-                    && !(&versions[version] == "pie")
-                    && !(&versions[version] == "pie_plus")
-                {
-                    return_value.push_str("Unsupported Version has been pushed!");
-                }
-            }
-        }
-    } else {
-        for json_value in 0..json_iter.len() {
-            if json_iter[json_value]["country"].is_empty() {
-                return_value.push_str("I cant find maintainer country!");
-            }
-
-            if json_iter[json_value]["github_username"].is_empty() {
-                return_value.push_str("I cant find maintainer github username!");
-            }
-
-            let devices = json_iter[json_value]["devices"].clone();
-
-            for device in 0..devices.len() {
-                if devices[device]["codename"].is_empty() {
-                    return_value.push_str("I cant find maintainer github username!");
-                }
-
-                let versions = devices[device]["versions"].clone();
-
-                for version in 0..versions.len() {
-                    if !(&versions[version] == "eleven")
-                        && !(&versions[version] == "ten")
-                        && !(&versions[version] == "ten_plus")
-                        && !(&versions[version] == "pie")
-                        && !(&versions[version] == "pie_plus")
-                    {
-                        return_value.push_str("Unsupported Version has been pushed!");
-                    }
-                }
-            }
-        }
-    }
-    return return_value;
-}
-
-fn main() {
-    let data = fs::read_to_string("official_devices/devices.json").expect("Unable to read file");
-    println!("Running Integrity Tests on official_devices/devices.json.....");
-    fs::write("/tmp/current_file", "devices.json").expect("Unable to write file");
-    let parsed = json::parse(&data).unwrap_or_else(|error| {
-        println!("\tError parsing JSON: {}", error);
-        fs::write("/tmp/error_message", error.to_string()).expect("Unable to write file");
-        panic!("Invalid JSON");
-    });
-    let resp = test_integrity_devices(parsed.clone());
-
-    if !resp.is_empty() {
-        println!("Integrity Tests Failed!");
-        fs::write("/tmp/error_message", resp.to_string()).expect("Unable to write file");
-        panic!("{}", resp);
-    } else {
-        let stringifed = json::stringify_pretty(parsed, 3);
-        fs::write("official_devices/devices.json", stringifed).expect("Unable to write file");
-    }
-
-    let entries = fs::read_dir("official_devices/team/").unwrap();
-
-    for entry in entries {
-        let path = entry.unwrap().path();
-        println!("Running Integrity Tests on {:?}", path);
-        let current_entry = path.clone();
-        let current_file = current_entry
-            .clone()
-            .into_os_string()
-            .into_string()
-            .unwrap();
-        let data2 = fs::read_to_string(path).expect("Unable to read file");
-        fs::write("/tmp/current_file", current_file).expect("Unable to write file");
-        let parsed_ = json::parse(&data2).unwrap_or_else(|error| {
-            println!("\tError parsing JSON: {}", error);
-            fs::write("/tmp/error_message", error.to_string()).expect("Unable to write file");
-            panic!("Invalid JSON");
-        });
-        let resp2 = test_integrity_team(parsed_.clone());
-
-        if !resp2.is_empty() {
-            println!("Integrity Tests Failed!");
-            fs::write("/tmp/error_message", resp.to_string()).expect("Unable to write file");
-            panic!("{}", resp2);
-        } else {
-            let stringifed = json::stringify_pretty(parsed_, 3);
-            fs::write(current_entry, stringifed).expect("Unable to write file");
-        }
-    }
-
-    fs::remove_file("/tmp/current_file").unwrap();
-
-    println!("Success!");
-}

runner.sh

@@ -33,28 +33,24 @@ function approvePR() {
     curl -s -X POST -d "{\"commit_id\": \"${COMMIT_HASH}\", \"event\": \"APPROVE\"}" -H "Authorization: token $GH_PERSONAL_TOKEN" "https://api.github.com/repos/PixelExperience/official_devices/pulls/$GIT_PR_NUMBER/reviews" > /dev/null
 }
 
-function checkLint() {
+function CheckCommit() {
     if [[ "$COMMIT_MESSAGE" =~ "[PIXEL-CI]" ]]; then
         if [[ -n "$GIT_PR_NUMBER" ]]; then
             commentPR "Please remove \"[PIXEL-CI]\" from your commit message."
             exit 0
         else
-            echo "Commit already linted, nothing to do."
+            echo "Commit already formatted, nothing to do."
             exit 0
         fi
     fi
 }
 
-function checkJsons() {
-    cargo run --release
+function Validator() {
+    python3 validator.py
     RESULT=$?
-    if [ "$RESULT" -ne 0 ]; then
-        ERROR_MESSAGE="$(cat /tmp/error_message)"
-        FAILED_FILE="$(cat /tmp/current_file)"
-    fi
     if [ -n "$GIT_PR_NUMBER" ]; then
         if [ "$RESULT" -ne 0 ]; then
-            message="JSON validation failed\nFile: ${FAILED_FILE}\nError message: ${ERROR_MESSAGE}"
+            message="JSON validation failed. Please check ${BUILD_URL} to find possible error."
             commentPR "$message"
             echo -e "$message"
         else
@@ -63,7 +59,7 @@ function checkJsons() {
             echo -e "$message"
         fi
     elif [ "$RESULT" -ne 0 ]; then
-        sendTG "Someone has merged a failing file. Please look in ASAP.\nError message: ${ERROR_MESSAGE}\nFile: ${FAILED_FILE}\n${ADMINS}"
+        sendTG "Someone has merged a failing file. Please check console to find possible error.\n\n${ADMINS}"
     else
         pushToGit
     fi
@@ -71,7 +67,6 @@ function checkJsons() {
 
 function pushToGit() {
     if [ -z "$GIT_PR_NUMBER" ] && [ -n "$(${GIT_CMD} status -s)" ]; then
-        ${GIT_CMD} pull --no-edit origin master
         ${GIT_CMD} add --all
         ${GIT_CMD} commit -m "[PIXEL-CI]: Lint files"
         ${GIT_CMD} remote set-url origin "https://pixelexperiencebot:${GH_PERSONAL_TOKEN}@github.com/PixelExperience/official_devices.git"
@@ -90,5 +85,5 @@ function pushToGit() {
     fi
 }
 
-checkLint
-checkJsons
+checkCommit
+Validator

validator.py

+import operator
+import json
+import sys
+import os.path
+import requests
+
+SUGGESTIONS = []
+ALLOWED_VERSIONS = ['eleven', 'ten', 'ten_plus', 'pie', 'pie_plus']
+WIKI_BASE_URL = 'https://raw.githubusercontent.com/PixelExperience/wiki/master/_data/devices'
+
+
+def test_integrity_devices():
+    try:
+        devices = json.loads(open('devices.json', 'r').read())
+    except:
+        SUGGESTIONS.append('devices.json is not a valid JSON.')
+        return
+
+    for device in devices:
+        device_index = devices.index(device)
+        if 'name' not in device:
+            SUGGESTIONS.append(
+                f"'name' key not found in devices.json for device index value {device_index}")
+
+        if 'brand' not in device:
+            SUGGESTIONS.append(
+                f"'brand' key not found in devices.json for device index value {device_index}")
+
+        if 'codename' not in device:
+            SUGGESTIONS.append(
+                f"'codename' key not found in devices.json for device index value {device_index}")
+        else:
+            codename = device.get('codename')
+            if not os.path.isfile(f"images/{codename}.png"):
+                SUGGESTIONS.append(
+                    f"images/{codename}.png not found for codename {codename} in devices.json for device index value {device_index}")
+            yaml = requests.head(f"{WIKI_BASE_URL}/{codename}.yml")
+            if yaml.status_code != 200:
+                SUGGESTIONS.append(
+                    f"Wiki YAML not found for codename {codename} in devices.json for device index value {device_index}")
+
+        if 'supported_versions' not in device:
+            SUGGESTIONS.append(
+                f"'supported_versions' key not found in devices.json for index value {device_index}")
+            continue
+
+        supported_versions = device['supported_versions']
+        for supported_version in supported_versions:
+            version_index = supported_versions.index(supported_version)
+            if 'version_code' not in supported_version:
+                SUGGESTIONS.append(
+                    f"'version_code' key not found in devices.json for device index value {device_index} and version index value {version_index}")
+
+            if supported_version['version_code'] not in ALLOWED_VERSIONS:
+                SUGGESTIONS.append(
+                    f"'version_code' key has a value that is not allowed in devices.json for device index value {device_index} and version index value {version_index}")
+    return
+
+
+def test_integrity_team():
+    try:
+        lead = json.loads(open('team/lead.json', 'r').read())
+    except:
+        SUGGESTIONS.append('team/lead.json is not a valid JSON.')
+        return
+
+    try:
+        core = json.loads(open('team/core.json', 'r').read())
+    except:
+        SUGGESTIONS.append('team/core.json is not a valid JSON.')
+        return
+
+    try:
+        maintainers = json.loads(open('team/maintainers.json', 'r').read())
+    except:
+        SUGGESTIONS.append('team/maintainers.json is not a valid JSON.')
+        return
+
+    for member in maintainers:
+        member_index = maintainers.index(member)
+        if 'name' not in member:
+            SUGGESTIONS.append(
+                f"'name' key not found in maintainers.json for member index value {member_index}")
+
+        if 'country' not in member:
+            SUGGESTIONS.append(
+                f"'country' key not found in maintainers.json for member index value {member_index}")
+
+        if 'github_username' not in member:
+            SUGGESTIONS.append(
+                f"'github_username' key not found in maintainers.json for member index value {member_index}")
+
+        if 'devices' not in member:
+            SUGGESTIONS.append(
+                f"'devices' key not found in maintainers.json for member index value {member_index}")
+            continue
+
+        devices = member['devices']
+        for device in devices:
+            device_index = devices.index(device)
+            if 'codename' not in device:
+                SUGGESTIONS.append(
+                    f"'codename' key not found in maintainers.json for member index value {member_index} and device index value {device_index}")
+
+            if 'versions' not in device:
+                SUGGESTIONS.append(
+                    f"'versions' key not found in maintainers.json for member index value {member_index}")
+                continue
+
+            versions = device['versions']
+            for version in versions:
+                version_index = versions.index(version)
+
+                if version not in ALLOWED_VERSIONS:
+                    SUGGESTIONS.append(
+                        f"'versions' key has a value that is not allowed in maintainers.json for device index value {device_index} and version index value {version_index} and member index value {member_index}")
+
+    devices = lead['devices']
+    for device in devices:
+        device_index = devices.index(device)
+        if 'codename' not in device:
+            SUGGESTIONS.append(
+                f"'codename' key not found in lead.json  for device index value {device_index}")
+
+            if 'versions' not in device:
+                SUGGESTIONS.append(
+                    f"'versions' key not found in lead.json for device index value {device_index}")
+                continue
+
+            versions = device['versions']
+            for version in versions:
+                version_index = versions.index(version)
+
+                if version not in ALLOWED_VERSIONS:
+                    SUGGESTIONS.append(
+                        f"'versions' key has a value that is not allowed in lead.json for device index value {device_index} and version index value {version_index}")
+
+    if 'name' not in lead:
+        SUGGESTIONS.append(
+            f"'name' key not found in lead.json")
+
+    if 'country' not in lead:
+        SUGGESTIONS.append(
+            f"'country' key not found in lead.json")
+
+    if 'github_username' not in lead:
+        SUGGESTIONS.append(
+            f"'github_username' key not found in lead.json")
+
+    if 'devices' not in lead:
+        SUGGESTIONS.append(
+            f"'devices' key not found in lead.json")
+
+    for member in core:
+        member_index = core.index(member)
+        if 'name' not in member:
+            SUGGESTIONS.append(
+                f"'name' key not found in core.json for member index value {member_index}")
+
+        if 'country' not in member:
+            SUGGESTIONS.append(
+                f"'country' key not found in core.json for member index value {member_index}")
+
+        if 'github_username' not in member:
+            SUGGESTIONS.append(
+                f"'github_username' key not found in core.json for member index value {member_index}")
+
+        if 'devices' not in member:
+            SUGGESTIONS.append(
+                f"'devices' key not found in core.json for member index value {member_index}")
+            continue
+
+        devices = member['devices']
+        for device in devices:
+            device_index = devices.index(device)
+            if 'codename' not in device:
+                SUGGESTIONS.append(
+                    f"'codename' key not found in core.json for member index value {member_index} and device index value {device_index}")
+
+            if 'versions' not in device:
+                SUGGESTIONS.append(
+                    f"'versions' key not found in core.json for member index value {member_index}")
+                continue
+
+            versions = device['versions']
+            for version in versions:
+                version_index = versions.index(version)
+
+                if version not in ALLOWED_VERSIONS:
+                    SUGGESTIONS.append(
+                        f"'versions' key has a value that is not allowed in core.json for device index value {device_index} and version index value {version_index} and member index value {member_index}")
+
+    return
+
+
+def format_json():
+    # Load JSON's
+    devices = json.loads(open('devices.json', 'r').read())
+    lead = json.loads(open('team/lead.json', 'r').read())
+    core = json.loads(open('team/core.json', 'r').read())
+    maintainers = json.loads(open('team/maintainers.json', 'r').read())
+
+    # Sort JSON alphabatically with selected keys
+    devices = sorted(devices, key=operator.itemgetter('codename', 'brand'))
+    #lead = sorted(lead, key=operator.itemgetter('name', 'country'))
+    core = sorted(core, key=operator.itemgetter('name', 'country'))
+    maintainers = sorted(
+        maintainers, key=operator.itemgetter('name', 'country'))
+
+    # Dump them
+    devices = json.dumps(devices, indent=3, sort_keys=False)
+    lead = json.dumps(lead, indent=3, sort_keys=False)
+    core = json.dumps(core, indent=3, sort_keys=False)
+    maintainers = json.dumps(maintainers, indent=3, sort_keys=False)
+
+    # Open JSON's
+    devices_json = open('devices.json', 'w')
+    lead_json = open('team/lead.json', 'w')
+    core_json = open('team/core.json', 'w')
+    maintainers_json = open('team/maintainers.json', 'w')
+
+    # Write dump's
+    devices_json.write(devices + '\n')
+    lead_json.write(lead + '\n')
+    core_json.write(core + '\n')
+    maintainers_json.write(maintainers + '\n')
+
+    # Close JSON's
+    devices_json.close()
+    lead_json.close()
+    core_json.close()
+    maintainers_json.close()
+
+    return 0
+
+
+def main():
+    print('Running Integrity tests for all the JSON\'s.')
+    test_devices = test_integrity_devices()
+    test_team = test_integrity_team()
+
+    if len(SUGGESTIONS) > 0:
+        print('Integrity test for one or more JSON\'s failed. Cannot proceed with JSON formatter.\n\n')
+        print('Below might be the reasons for test failure.')
+        for SUGGESTION in SUGGESTIONS:
+            print(SUGGESTION)
+        sys.exit(1)
+    else:
+        print('Integrity test passed successfully. Now formatting all the JSON\'s')
+        format_json()
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()