ZBIO-513 Update retriever to retrieve articles from fetcher

collaborate/Makefile

@@ -11,10 +11,12 @@ dockerise: build
 	${MAKE} clean
 
 deploy:
-	@kubectl apply -f kubernetes-manifests/deploy.yaml 
+	@kubectl apply -f kubernetes-manifests/fetcher.yaml 
+	@kubectl apply -f kubernetes-manifests/retriever.yaml 
 
 undeploy:
-	@kubectl delete -f kubernetes-manifests/deploy.yaml >/dev/null 2>&1 || true
+	@kubectl delete -f kubernetes-manifests/fetcher.yaml >/dev/null 2>&1 || true
+	@kubectl delete -f kubernetes-manifests/retriever.yaml >/dev/null 2>&1 || true
 
 clean:
 	-@rm -rf bin/*

collaborate/README.md

@@ -1,6 +1,6 @@
 # Dependent microservices for collaborate feature in ROOST
 
-Two containerised services `fetcher` and `retriver` run in kubernetes environment in single pod but in two containers. `fetcher` services expose an api at <http://roost-master:30047/articles?tag=kubernetes> to fetch latest articles from <http://dev.to> and stores in provisioned pod volume. `retriver` service is dependent on `fetcher` service to create file where fetched articles are stored. As soon as articles file is found, `retriever` service reads the content and sends output over HTTP/browser.
+Two containerised services `fetcher` and `retriver` run in kubernetes environment in two pods. `fetcher` services expose an api at <http://roost-master:30047/articles?tag=kubernetes> to fetch latest articles from <http://dev.to> based on provided tags and stores in provisioned pod volume. `retriver` service is dependent on `fetcher` service to retrieve stored articles in `fetcher` service volume. As soon as articles file is found, `retriever` service reads the content and sends output over HTTP/browser.
 
 ## Commands to deploy
 
@@ -26,20 +26,20 @@ make dockerise
 2. `fetcher` service must be running
 
     ```bash
-        kubectl logs pod collaborate fetcher
+        kubectl logs pod collab-fetcher
     ```
 
 3. Make `curl` request to fetcher service to retrive articles which save articles into file
 
     ```bash
         curl "http://roost-master:30047/articles?tag=kubernetes"
-        kubectl logs pod collaborate fetcher
+        kubectl logs pod collab-fetcher
     ```
 
 4. Above content must be served by `retriever` service.
 
     ```bash
         curl "http://roost-master:30048/"
         # --tail n: shows last n lines from logs
-        kubectl logs collaborate retriever --tail 400
+        kubectl logs collab-retriever --tail 400
     ```

collaborate/cmd/fetcher/main.go

@@ -78,7 +78,7 @@ func articleHandler(w http.ResponseWriter, r *http.Request) {
 
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		log.Fatalf("Error reading response from dev.to api. Error: %v", err)
+		log.Fatalf("error reading response from dev.to api. Error: %v", err)
 	}
 
 	var d bytes.Buffer
@@ -106,12 +106,33 @@ func cleanHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte("Sucessfully deleted article store"))
 }
 
+func serveHandler(w http.ResponseWriter, r *http.Request) {
+	outputPath := getFilePath()
+
+	content, err := readArticles(outputPath)
+	if err != nil {
+		log.Printf("\nerror reading articles from file. error: %v", err)
+		w.Header().Add("Content-Type", "text/plain")
+		w.Write([]byte("Content not available"))
+		return
+	}
+	w.Header().Add("Content-Type", "application/json")
+	log.Printf("Successfully retrieved articles...")
+	w.Write(content)
+}
+
+func readArticles(src string) ([]byte, error) {
+	data, err := ioutil.ReadFile(src)
+	return data, err
+}
+
 func main() {
 	log.Printf("Listening on :8080")
 
-	http.HandleFunc("/", rootHandler)
+	// http.HandleFunc("/", rootHandler)
 	http.HandleFunc("/articles", articleHandler)
 	http.HandleFunc("/clean", cleanHandler)
+	http.HandleFunc("/serve", serveHandler)
 
 	log.Fatal(http.ListenAndServe(":8080", nil))
 }

collaborate/cmd/retriever/main.go

@@ -1,15 +1,15 @@
 package main
 
 import (
+	"fmt"
 	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
 	"os"
-	"path/filepath"
+	"time"
 )
 
-var inputFilename string = "devio_articles.json"
 var appPort = "8081"
 
 func main() {
@@ -20,25 +20,37 @@ func main() {
 }
 
 func articleHandler(w http.ResponseWriter, r *http.Request) {
-	basePath, ok := os.LookupEnv("STORAGE_DIR")
-	if !ok {
-		log.Fatalf(" ENV[STORAGE_DIR] missing.")
+	fetcherEndpoint := os.Getenv("FETCH_SERVICE")
+	if fetcherEndpoint == "" {
+		log.Fatal("Missing fetcher service endpoint. Set ENV[FETCH_SERVICE] to continue")
 	}
-	inputFilepath := filepath.Join(basePath, inputFilename)
-
-	content, err := readArticles(inputFilepath)
+	endpoint := "http://" + fetcherEndpoint + "/serve"
+	req, _ := http.NewRequest(http.MethodGet, endpoint, nil)
+	resp, err := doRequest(req)
 	if err != nil {
-		log.Printf("\nerror reading articles from file. error: %v", err)
-		w.Header().Add("Content-Type", "text/plain")
-		w.Write([]byte("Content not available"))
+		log.Printf("unable to get articles from the service. Error: %v", err)
+		w.WriteHeader(400)
+		w.Write([]byte(fmt.Sprintf("No articles retrieved. Reason: %v", err.Error())))
 		return
 	}
-	w.Header().Add("Content-Type", "application/json")
-	log.Printf("Successfully retrieved articles...")
-	w.Write(content)
+	defer resp.Body.Close()
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Fatalf("error reading response from service. Error: %v", err)
+	}
+
+	w.Write(data)
 }
 
-func readArticles(src string) ([]byte, error) {
-	data, err := ioutil.ReadFile(src)
-	return data, err
+func doRequest(r *http.Request) (*http.Response, error) {
+	client := http.Client{
+		Timeout: time.Second * 10,
+	}
+	resp, err := client.Do(r)
+	if err != nil {
+		log.Printf("error requesting to fetcher service at endpoint. error: %v", err)
+		return nil, err
+	}
+	return resp, err
 }

collaborate/kubernetes-manifests/fetcher.yaml

@@ -0,0 +1,43 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: collab-fetcher
+  labels:
+    app: fetcher
+spec:
+  containers:
+  - name: fetcher
+    image: roost-example/fetcher:v1
+    env:
+    - name: STORAGE_DIR
+      value: /app-store
+    volumeMounts:
+    - mountPath: /app-store
+      name: article-storage
+    resources:
+      limits:
+        memory: "128Mi"
+        cpu: "200m"
+    ports:
+      - name: fetcher-port
+        containerPort: 8080
+    imagePullPolicy: Never
+  terminationGracePeriodSeconds: 0
+  volumes:
+  - name: article-storage
+    emptyDir: {}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: fetcher
+spec:
+  type: NodePort
+  selector:
+    app: fetcher
+  ports:
+  - name: fetch-svc-port
+    port: 30047
+    targetPort: 8080
+    nodePort: 30047

collaborate/kubernetes-manifests/retriever.yaml

@@ -0,0 +1,37 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: collab-retriever
+  labels:
+    app: retriever
+spec:
+  containers:
+  - name: retriever
+    image: roost-example/retriever:v1
+    env:
+    - name: FETCH_SERVICE
+      value: fetcher:30047
+    resources:
+      limits:
+        memory: "128Mi"
+        cpu: "200m"
+    ports:
+      - name: retriever-port
+        containerPort: 8081
+    imagePullPolicy: Never
+  terminationGracePeriodSeconds: 0
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: retriever
+spec:
+  type: NodePort
+  selector:
+    app: retriever
+  ports:
+  - name: retriever-port
+    port: 30048
+    targetPort: 8081
+    nodePort: 30048