Dockerhub, Appsody releases, and star/watcher/forks for /stacks and /appsody repos

.gitignore

@@ -0,0 +1,3 @@
+/metrics/appsody_reports
+/metrics/node_modules
+.DS_Store

README.md

@@ -1,2 +1,33 @@
 # utils
+
+## metrics
 Repository to store various utilities related to Appsody project
+
+
+### Data Pulling scripts
+
+In order to execute any of these scripts, you'll need to first export your GitHub API Key to an environment variable called `GITHUB_API_KEY`.
+
+ - run_all.sh
+
+ Usage:
+ - Can be passed a date in the form of `YYY-MM-DD`. This date would have to correspond to a folder in `metrics/appsody_reports/` which contains data from a previous run, to provide the total amount downloads (dockerhub & GitHub), or gained/lost (stars/watchers/forks) since that date. 
+
+ - For example: `sh run_all.sh 2020-02-10` would give the results of the current most up-to-date information, as well as a data compared to the results on the 10th of February 2020.
+
+ Runs all of the `.js` scripts in this folder consecutively, then runs `../data_pulling_utils/json_report_to_csv.py` which creates a `.csv` file for any comparisons again'st the given date (if the `.json` files from that date are found).
+
+ - appsody_dockerhub.js
+
+ Gets the pull count for each of the repositories on [appsody's dockerhub](https://hub.docker.com/u/appsody), and when the repository was last updated.
+
+ - appsody_releases.js
+
+ Gets the the download count for each of the binaries for every release of appsody from [the GitHub releases](https://github.com/appsody/appsody/releases).
+
+ - appsody_stars_watchers_forks.js
+
+ Gets the number of stars, watchers, and forks for the [appsody/stacks repository](https://github.com/appsody/stacks) and the [appsody/appsody repository](https://github.com/appsody/appsody).
+
+
+All of the results are stored in an `appsody_reports/DATE/` folder.

metrics/data_pulling_scripts/appsody_dockerhub.js

@@ -0,0 +1,43 @@
+const axios = require('axios');
+const tools = require('../data_pulling_utils/data_pulling_tools');
+
+const args = process.argv.slice(2)
+
+retrieveData();
+
+ async function retrieveData() {
+
+
+ const AuthStr = `token ${process.env.GITHUB_API_KEY}`
+ const URL = 'https://hub.docker.com/v2/repositories/appsody/?page=1&page_size=100';
+
+ axios.get(URL, { headers: { Authorization: AuthStr } })
+ .then(response => {
+
+ dockerHubResultsArray = [];
+ for (let item of response.data.results) {
+
+ const date = new Date(item.last_updated)
+ var readableTimestamp = tools.addZeroPrefix(date.getDate()) + "-" + tools.addZeroPrefix((date.getMonth() +1)) + "-" + tools.addZeroPrefix(date.getFullYear());
+
+ const single = {
+ "name": item.name,
+ "pull_count": item.pull_count,
+ "last_updated": readableTimestamp
+ }
+ dockerHubResultsArray.push(single);
+ }
+
+ tools.createLogFile("dockerhub_appsody.json", dockerHubResultsArray, function(err) {
+ console.log(err);
+ });
+
+ if(args[0] != undefined) {
+ tools.createComparisonFile("dockerhub_appsody", dockerHubResultsArray, "name", args[0]);
+ }
+
+ })
+ .catch((error) => {
+ console.log('error ' + error);
+ });
+ }

metrics/data_pulling_scripts/appsody_releases.js

@@ -0,0 +1,44 @@
+const axios = require('axios');
+const tools = require('../data_pulling_utils/data_pulling_tools');
+
+const args = process.argv.slice(2)
+
+retrieveData();
+
+ async function retrieveData() {
+
+
+ const AuthStr = `token ${process.env.GITHUB_API_KEY}`
+ const URL = 'https://api.github.com/repos/appsody/appsody/releases';
+
+ axios.get(URL, { headers: { Authorization: AuthStr } })
+ .then(response => {
+ releaseResultsArray = [];
+ for (let item of response.data) {
+
+ let name = item.tag_name
+
+ for (let asset of item.assets) {
+ const single = {
+ "release": name,
+ "cli_binary": asset.name,
+ "download_count": asset.download_count
+ }
+
+ releaseResultsArray.push(single);
+ }
+ }
+
+ tools.createLogFile("releases_appsody.json", releaseResultsArray, function(err) {
+ console.log(err);
+ });
+
+ if(args[0] != undefined) {
+ tools.createComparisonFile("releases_appsody", releaseResultsArray, "cli_binary", args[0]);
+ }
+
+ })
+ .catch((error) => {
+ console.log('error ' + error);
+ });
+ }

metrics/data_pulling_scripts/appsody_stars_watchers_forks.js

@@ -0,0 +1,79 @@
+const { graphql } = require('@octokit/graphql')
+const tools = require('../data_pulling_utils/data_pulling_tools');
+
+const args = process.argv.slice(2)
+
+async function asyncForEach(array, callback) {
+ for (let index = 0; index < array.length; index++) {
+ await callback(array[index], index, array);
+ }
+}
+
+const queryTotals = async (repo, org) => {
+ const graphqlWithAuth = graphql.defaults({
+ headers: {
+ authorization: `token ${process.env.GITHUB_API_KEY}`
+ }
+ })
+ let data
+ try {
+ data = await graphqlWithAuth(`
+ {
+ repository(name: "${repo}", owner: "${org}") {
+ forks {
+ totalCount
+ }
+ stargazers {
+ totalCount
+ }
+ watchers {
+ totalCount
+ }
+ }
+ }`)
+ } catch (err) {
+ if (err.name === 'HttpError') {
+ setTimeout(() => {
+ queryTotals(repo, org)
+ }, 10000)
+ } else {
+ throw (err)
+ }
+ } finally {
+
+ const single = {
+ "repo": repo,
+ "stars": data.repository.stargazers.totalCount,
+ "watchers": data.repository.watchers.totalCount,
+ "forks": data.repository.forks.totalCount,
+ }
+ return single;
+ }
+}
+
+var queries = [
+ ['appsody', 'stacks'],
+ ['appsody', 'appsody']
+];
+results = [];
+
+const callQueries = async () => {
+
+ const start = async () => {
+ await asyncForEach(queries, async (q) => {
+ let res = await (queryTotals(q[1], q[0]));
+ results.push(res)
+ });
+
+ tools.createLogFile(`stars_watchers_forks.json`, results, function(err) {
+ console.log(err);
+ });
+
+ if(args[0] != undefined) {
+ tools.createComparisonFile("stars_watchers_forks", results, "repo", args[0]);
+ }
+ }
+ start();
+}
+
+callQueries();

metrics/data_pulling_scripts/run_all.sh

@@ -0,0 +1,14 @@
+COMPARISON=$1
+
+for entry in ./*.js
+do
+ echo "$entry"
+ if [ -z "$COMPARISON" ]
+ then
+ `node $entry`
+ else
+ echo "Comparison $COMPARISON was given"
+ `node $entry $COMPARISON`
+ fi
+done
+`python ../data_pulling_utils/json_report_to_csv.py ../appsody_reports/`

metrics/data_pulling_utils/data_pulling_tools.js

@@ -0,0 +1,164 @@
+const fs = require('fs');
+const moment = require('moment');
+
+const folderPath = "../appsody_reports/"
+
+module.exports = {
+ createLogFile: function(extension, array, cb) {
+
+ ensurePathExists(folderPath, 0744, function(err) {
+ if (err) {
+ cb(err)
+ }
+ });
+
+ d = new Date().toISOString().slice(0, 10);
+
+ var filePath = `${folderPath}${d}/`
+ ensurePathExists(filePath, 0744, function(err) {
+ if (err) {
+ cb(err)
+ } else {
+ var filename = `${filePath}${extension}`
+
+ writeFile(filename, array, function(err) {
+ if (err) {
+ cb(err)
+ }
+ });
+ }
+ })
+ },
+ addZeroPrefix: function(num) {
+ if (num.toString().length < 2) {
+ return `0${num}`;
+ } else {
+ return num;
+ }
+ },
+ createComparisonFile: function(filename, newRes, id, previous) {
+ oldResPath = `../appsody_reports/${previous}/${filename}.json`;
+ fs.access(oldResPath, fs.F_OK, (err) => {
+ if (err) {
+ console.error("No file found to make comparison.")
+ return
+ }
+
+ let rawdata = fs.readFileSync(oldResPath);
+ let oldRes = JSON.parse(rawdata); 
+
+ var comparison = compareToLast(oldRes, newRes, id);
+ this.createLogFile(`${filename}_comparison.json`, comparison, function(err) {
+ console.log(err);
+ });
+ })
+ }
+};
+
+function ensurePathExists(path, mask, cb) {
+ if (typeof mask == 'function') {
+ cb = mask;
+ mask = 0777;
+ }
+ fs.mkdir(path, mask, function(err) {
+ if (err) {
+ if (err.code == 'EEXIST') {
+ cb(null);
+ } else {
+ cb(err);
+ }
+ } else {
+ cb(null);
+ }
+ });
+}
+
+function writeFile(filename, array, cb) {
+ fs.writeFile(filename, JSON.stringify(array, null, 4), 'utf8', (err) => {
+ if (err) {
+ console.log("An error occured while writing JSON Object to File.");
+ cb(err);
+ } else {
+ cb(null);
+ }
+ });
+}
+
+function compareToLast(oldRes, newRes, id) {
+ comparisons = [];
+
+ for (let i = 0; i < newRes.length; i++) {
+ let single = {};
+ switch (id) {
+ case "repo":
+
+ var noMatch = true;
+ for (let j = 0; j < oldRes.length; j++) {
+ if (newRes[i].repo === oldRes[j].repo) {
+ noMatch = false;
+
+ single = {
+ "repo": newRes[i].repo,
+ "stars": newRes[i].stars - oldRes[j].stars,
+ "watchers": newRes[i].watchers - oldRes[j].watchers,
+ "forks": newRes[i].forks - oldRes[j].forks
+ }
+ }
+ }
+ if (noMatch) {
+ single = newRes[i];
+ }
+ comparisons.push(single);
+ break;
+ case "name":
+ var noMatch = true;
+ for (let j = 0; j < oldRes.length; j++) {
+ if (newRes[i].name === oldRes[j].name) {
+ noMatch = false;
+
+ single = {
+ "name": newRes[i].name,
+ "pull_count": newRes[i].pull_count - oldRes[j].pull_count,
+ "last_updated": newRes[i].last_updated
+ }
+ }
+ }
+ if (noMatch) {
+
+ single = {
+ "name": newRes[i].name,
+ "pull_count": newRes[i].pull_count,
+ "last_updated": newRes[i].last_updated
+ }
+ }
+
+ comparisons.push(single);
+ break;
+
+ case "cli_binary":
+ var noMatch = true;
+ for (let j = 0; j < oldRes.length; j++) {
+ if (newRes[i].cli_binary === oldRes[j].cli_binary && newRes[i].release === oldRes[j].release) {
+ noMatch = false;
+
+ single = {
+ "release": newRes[i].release,
+ "cli_binary": newRes[i].cli_binary,
+ "download_count": newRes[i].download_count - oldRes[j].download_count
+ }
+ }
+ }
+ if (noMatch) {
+ single = {
+ "release": newRes[i].release,
+ "cli_binary": newRes[i].cli_binary,
+ "download_count": newRes[i].download_count
+ }
+ }
+
+ comparisons.push(single);
+ break;
+ }
+ }
+ return comparisons;
+}