Dockerhub, Appsody releases, and star/watcher/forks for /stacks and /appsody repos

.gitignore

@@ -0,0 +1,3 @@
+/appsody_reports
+/node_modules
+.DS_Store

README.md

@@ -1,2 +1,27 @@
 # utils
 Repository to store various utilities related to Appsody project
+
+
+### Data Pulling scripts
+
+In order to execute any of these scripts, you'll need to first export your GitHub API Key to an environment variable called `GITHUB_API_KEY`.
+
+ - run_all.sh
+
+ Runs all of the `.js` scripts in this folder consecutively, then runs `../data_pulling_utils/json_to_csv.py` which creates a `.csv` file for any comparisons made against the previous month (if the `.json` files from last month are found). 
+ In order for the comparison to be made, the data from the previous month would have had to have been collected on the same day (e.g 01/12/19 and 01/01/2020); gathering data after the 28th day of the month wouldn't be recommended.
+
+ - appsody_dockerhub.js
+
+ Gets the pull count for each of the repositories on [appsody's dockerhub](https://hub.docker.com/u/appsody), and when the repository was last updated.
+
+ - appsody_releases.js
+
+ Gets the the download count for each of the binaries for every release of appsody from [the GitHub releases](https://github.com/appsody/appsody/releases).
+
+ - appsody_stars_watchers_forks.js
+
+ Gets the number of stars, watchers, and forks for the [appsody/stacks repository](https://github.com/appsody/stacks) and the [appsody/appsody repository](https://github.com/appsody/appsody).
+
+
+All of the results are stored in an `appsody_reports/DATE/` folder.

data_pulling_scripts/appsody_dockerhub.js

@@ -0,0 +1,41 @@
+const axios = require('axios');
+const tools = require('../data_pulling_utils/tools');
+
+
+
+retrieveData();
+
+ async function retrieveData() {
+
+
+ const AuthStr = `token ${process.env.GITHUB_API_KEY}`
+ const URL = 'https://hub.docker.com/v2/repositories/appsody/?page=1&page_size=100';
+
+ axios.get(URL, { headers: { Authorization: AuthStr } })
+ .then(response => {
+
+ dockerHubResultsArray = [];
+ for (let item of response.data.results) {
+
+ const date = new Date(item.last_updated)
+ var readableTimestamp = tools.addZeroPrefix(date.getDate()) + "-" + tools.addZeroPrefix((date.getMonth() +1)) + "-" + tools.addZeroPrefix(date.getFullYear());
+
+ const single = {
+ "name": item.name,
+ "pull_count": item.pull_count,
+ "last_updated": readableTimestamp
+ }
+ dockerHubResultsArray.push(single);
+ }
+
+ tools.createLogFile("dockerhub_appsody.json", dockerHubResultsArray, function(err) {
+ console.log(err);
+ });
+
+ tools.createComparisonFile("dockerhub_appsody", dockerHubResultsArray, "name");
+
+ })
+ .catch((error) => {
+ console.log('error ' + error);
+ });
+ }

data_pulling_scripts/appsody_releases.js

@@ -0,0 +1,40 @@
+const axios = require('axios');
+const tools = require('../data_pulling_utils/tools');
+
+retrieveData();
+
+ async function retrieveData() {
+
+
+ const AuthStr = `token ${process.env.GITHUB_API_KEY}`
+ const URL = 'https://api.github.com/repos/appsody/appsody/releases';
+
+ axios.get(URL, { headers: { Authorization: AuthStr } })
+ .then(response => {
+ releaseResultsArray = [];
+ for (let item of response.data) {
+
+ let name = item.tag_name
+
+ for (let asset of item.assets) {
+ const single = {
+ "release": name,
+ "cli_binary": asset.name,
+ "download_count": asset.download_count
+ }
+
+ releaseResultsArray.push(single);
+ }
+ }
+
+ tools.createLogFile("releases_appsody.json", releaseResultsArray, function(err) {
+ console.log(err);
+ });
+
+ tools.createComparisonFile("releases_appsody", releaseResultsArray, "cli_binary");
+
+ })
+ .catch((error) => {
+ console.log('error ' + error);
+ });
+ }

data_pulling_scripts/appsody_stars_watchers_forks.js

@@ -0,0 +1,76 @@
+const { graphql } = require('@octokit/graphql')
+const tools = require('../data_pulling_utils/tools');
+
+
+async function asyncForEach(array, callback) {
+ for (let index = 0; index < array.length; index++) {
+ await callback(array[index], index, array);
+ }
+}
+
+const queryTotals = async (repo, org) => {
+ const graphqlWithAuth = graphql.defaults({
+ headers: {
+ authorization: `token ${process.env.GITHUB_API_KEY}`
+ }
+ })
+ let data
+ try {
+ data = await graphqlWithAuth(`
+ {
+ repository(name: "${repo}", owner: "${org}") {
+ forks {
+ totalCount
+ }
+ stargazers {
+ totalCount
+ }
+ watchers {
+ totalCount
+ }
+ }
+ }`)
+ } catch (err) {
+ if (err.name === 'HttpError') {
+ setTimeout(() => {
+ queryTotals(repo, org)
+ }, 10000)
+ } else {
+ throw (err)
+ }
+ } finally {
+
+ const single = {
+ "repo": repo,
+ "stars": data.repository.stargazers.totalCount,
+ "watchers": data.repository.watchers.totalCount,
+ "forks": data.repository.forks.totalCount,
+ }
+ return single;
+ }
+}
+
+var queries = [
+ ['appsody', 'stacks'],
+ ['appsody', 'appsody']
+];
+results = [];
+
+const callQueries = async () => {
+
+ const start = async () => {
+ await asyncForEach(queries, async (q) => {
+ let res = await (queryTotals(q[1], q[0]));
+ results.push(res)
+ });
+
+ tools.createLogFile(`stars_watchers_forks.json`, results, function(err) {
+ console.log(err);
+ });
+
+ tools.createComparisonFile("stars_watchers_forks", results, "repo");
+ }
+ start();
+}
+
+callQueries();

data_pulling_scripts/run_all.sh

@@ -0,0 +1,6 @@
+for entry in ./*.js
+do
+ echo "$entry"
+ `node $entry`
+done
+`python ../data_pulling_utils/json_to_csv.py ../appsody_reports/`

data_pulling_utils/json_to_csv.py

@@ -0,0 +1,34 @@
+from os import walk, path, remove
+
+from datetime import date 
+
+import csv, json, sys, logging
+
+
+if sys.argv[1] is not None:
+
+ today = date.today().isoformat()
+ targetFolder = sys.argv[1] + today + "/"
+
+ f = []
+ for (dirpath, dirnames, filenames) in walk(targetFolder):
+ f.extend(filenames)
+ break
+
+ for file in f:
+ if ".csv" not in file and "_comparison" in file:
+ fileInput = targetFolder + file
+ fileOutput = fileInput[:-5] + ".csv"
+
+ inputFile = open(fileInput)
+ outputFile = open(fileOutput, 'w')
+ data = json.load(inputFile)
+ inputFile.close()
+
+ output = csv.writer(outputFile)
+
+ output.writerow(data[0].keys()) # header row
+
+ for row in data:
+ output.writerow(row.values())
+ output.writerow("")