#!/usr/bin/env python3 """ This script can be used to update this repository with the latest information. It performs the following actions: - Updates the local clone of the repository - Downloads the latest setup files from Appveyor - Updates the README with the latest version info (commit ID and message) - Uploads changes to the repository """ import os import sys import pickle import json from subprocess import call import requests from requests.exceptions import ConnectionError as ConnError ETAGFILE = "etags" ETAGS = {} # type: dict DOC = __doc__.replace("This script", f"The [{__file__}]({__file__}) script") def load_etags(): """ Read in etags file and populates dictionary. """ try: with open(ETAGFILE, "rb") as etagfile: ETAGS.update(pickle.load(etagfile)) except FileNotFoundError: # print("--> No etags file found. Skipping load.") pass def save_etags(): """ Save (potentially new) etags to file. """ with open(ETAGFILE, "wb") as etagfile: pickle.dump(ETAGS, etagfile) def download(url, fname=None): """ Download a URL if necessary. If the URL's etag matches the existing one, the download is skipped. """ if fname is None: fname = url.split("/")[-1] print("--> Downloading {} → {}".format(url, fname)) try: req = requests.get(url, stream=True) except ConnError: print("Error while trying to download {}".format(url), file=sys.stderr) print("Skipping.", file=sys.stderr) return size = int(req.headers.get("content-length")) etag = req.headers.get("etag") oldet = ETAGS.get(url) if etag == oldet and os.path.exists(fname): fod_size = os.path.getsize(fname) if fod_size == size: print("File already downloaded. Skipping.", end="\n\n") return fname ETAGS[url] = etag prog = 0 with open(fname, "wb") as dlfile: for chunk in req.iter_content(chunk_size=256): dlfile.write(chunk) prog += len(chunk) print("\r{:2.1f}%".format(prog / size * 100), end="", flush=True) print("\nDone!") print() return fname def get_appveyor_info(): # TODO: Check what happens when a build is in progress and so has no # available artifacts apiurl = "https://ci.appveyor.com/api/" account = "G-Node" project_name = "WinGIN" url = os.path.join(apiurl, "projects", account, project_name) r = requests.get(url) projects = json.loads(r.text) build = projects["build"] info = dict() info["json"] = r.text info["commit"] = build["commitId"] info["message"] = build["message"] info["version"] = build["version"] dlurls = [] for job in build["jobs"]: # should just be one for this project if job["status"] == "success": artifacts_url = os.path.join(apiurl, "buildjobs", job["jobId"], "artifacts") r = requests.get(artifacts_url) artifacts = json.loads(r.text) for a in artifacts: dlurls.append(os.path.join(apiurl, "buildjobs", job["jobId"], "artifacts", a["fileName"])) info["artifacts"] = dlurls return info def update_readme(info): print(f"Latest commit: {info['message']} [{info['commit']}]") commiturl = (f"https://github.com/G-Node/WinGIN/commit/{info['commit']}") vertext = (f"\n## Version {info['version']}\n\n" "The current version of the setup files corresponds to:\n\n" f"- Commit ID: [{info['commit']}]({commiturl})\n" f"- Message: {info['message']}\n") scriptdesc = f"\n## {__file__}\n{DOC}" with open("instructions.md") as instructfile: instructions = instructfile.read() with open("README.md", "w") as readme: readme.write(instructions) readme.write(vertext) readme.write(scriptdesc) def update_verinfo(info): with open("version", "w") as verfile: verfile.write(info["version"]) with open("build.json", "w") as jsonfile: jsonfile.write(info["json"]) def main(): print("Updating local repository") call(["gin", "download", "--content"]) load_etags() dlfiles = [] avinfo = get_appveyor_info() artifacts = avinfo["artifacts"] print(f"Latest build message: {avinfo['message']}") if not len(artifacts): print("No artifacts in current build. Perhaps it is still running?") sys.exit("aborting") print(f"Found {len(artifacts)} artifacts") for url in artifacts: fname = None if url.endswith("setup.exe"): fname = "WinGIN-install.exe" dlfiles.append(download(url, fname)) save_etags() print("Updating README.md") update_readme(avinfo) update_verinfo(avinfo) print("Uploading changes") # at each run, we expect the following files to change changedfiles = ["README.md", "etags", "Setup.msi", "WinGIN-install.exe", "version", "build.json"] # any other changes (e.g., changes to this script) should be handled # manually call(["gin", "upload", *changedfiles]) if __name__ == "__main__": main()