From 51843d1321fdbb9197319b834a1a93ab88cd78ab Mon Sep 17 00:00:00 2001 From: Alessio Romano Date: Thu, 9 Jan 2025 16:11:25 +0100 Subject: [PATCH] Software Release --- Release/Client/client.yaml | 13 + Release/Client/client.zip | Bin 0 -> 2890 bytes Release/DAG | 172 +++++++ .../Datastore/DSLogic/DSlogcomp/.dockerignore | 4 + .../Datastore/DSLogic/DSlogcomp/Dockerfile | 42 ++ .../DSLogic/DSlogcomp/app/DBCalls.py | 60 +++ .../DSLogic/DSlogcomp/app/DBhooks.py | 107 ++++ .../Datastore/DSLogic/DSlogcomp/app/DSDesc.py | 34 ++ .../DSLogic/DSlogcomp/app/__init__.py | 49 ++ .../app/__pycache__/DBCalls.cpython-38.pyc | Bin 0 -> 958 bytes .../app/__pycache__/DBhooks.cpython-38.pyc | Bin 0 -> 1837 bytes .../app/__pycache__/DSDesc.cpython-38.pyc | Bin 0 -> 965 bytes .../app/__pycache__/__init__.cpython-38.pyc | Bin 0 -> 756 bytes .../app/__pycache__/config.cpython-38.pyc | Bin 0 -> 1160 bytes .../Datastore/DSLogic/DSlogcomp/app/config.py | 50 ++ .../DSLogic/DSlogcomp/app/main/__index__.py | 0 .../main/__pycache__/DBhooks.cpython-38.pyc | Bin 0 -> 1080 bytes .../main/__pycache__/DSDesc.cpython-38.pyc | Bin 0 -> 1172 bytes .../main/__pycache__/cleanview.cpython-38.pyc | Bin 0 -> 641 bytes .../main/__pycache__/descview.cpython-38.pyc | Bin 0 -> 483 bytes .../main/__pycache__/dlogview.cpython-38.pyc | Bin 0 -> 461 bytes .../main/__pycache__/dumpfview.cpython-38.pyc | Bin 0 -> 531 bytes .../main/__pycache__/errors.cpython-38.pyc | Bin 0 -> 360 bytes .../__pycache__/freezeview.cpython-38.pyc | Bin 0 -> 793 bytes .../main/__pycache__/indexview.cpython-38.pyc | Bin 0 -> 778 bytes .../app/main/__pycache__/main.cpython-38.pyc | Bin 0 -> 436 bytes .../main/__pycache__/statview.cpython-38.pyc | Bin 0 -> 456 bytes .../__pycache__/streamview.cpython-38.pyc | Bin 0 -> 762 bytes .../DSLogic/DSlogcomp/app/main/cleanview.py | 22 + .../DSLogic/DSlogcomp/app/main/descview.py | 14 + .../DSLogic/DSlogcomp/app/main/dlogview.py | 24 + .../DSLogic/DSlogcomp/app/main/dumpfview.py | 29 ++ .../DSLogic/DSlogcomp/app/main/errors.py | 6 + .../DSLogic/DSlogcomp/app/main/freezeview.py | 26 + .../DSLogic/DSlogcomp/app/main/indexview.py | 32 ++ .../DSLogic/DSlogcomp/app/main/main.py | 5 + .../DSLogic/DSlogcomp/app/main/statview.py | 23 + .../DSLogic/DSlogcomp/app/main/streamview.py | 38 ++ .../DSlogcomp/app/templates/index.html | 52 ++ .../Datastore/DSLogic/DSlogcomp/envsetup.sh | 31 ++ .../DSLogic/DSlogcomp/gunicorn.conf.py | 279 ++++++++++ .../DSLogic/DSlogcomp/requirements.txt | 17 + .../Datastore/DSLogic/kubernetes/depl.yaml | 47 ++ .../DataBuff/Databuffcomp/.dockerignore | 6 + .../DataBuff/Databuffcomp/Dockerfile | 46 ++ .../Databuffcomp/class/appmodel/appmodel.go | 17 + .../Databuffcomp/class/appstr/appstr.go | 263 ++++++++++ .../Databuffcomp/class/dsstat/dsstat.go | 81 +++ .../Databuffcomp/class/envdef/envdef.go | 45 ++ .../Databuffcomp/class/fbufstat/fbufstat.go | 100 ++++ .../Databuffcomp/class/fwrite/fwrite.go | 134 +++++ .../DataBuff/Databuffcomp/class/go.mod | 36 ++ .../DataBuff/Databuffcomp/class/go.sum | 95 ++++ .../Databuffcomp/class/neterr/neterr.go | 18 + .../Databuffcomp/class/netutil/netutil.go | 12 + .../class/simplelogger/simplelogger.go | 76 +++ .../Databuffcomp/main/adm/Dsstat.json | 1 + .../Databuffcomp/main/adm/StatDesc.json | 1 + .../DataBuff/Databuffcomp/main/envsetup.sh | 18 + .../DataBuff/Databuffcomp/main/go.mod | 42 ++ .../DataBuff/Databuffcomp/main/go.sum | 103 ++++ .../Databuffcomp/main/log/LogStream.json | 0 .../DataBuff/Databuffcomp/main/main.go | 165 ++++++ .../Databuffcomp/main/static/Dsstat.json | 1 + .../Databuffcomp/main/static/LogStream.json | 0 .../Databuffcomp/main/static/StatDesc.json | 1 + .../Datastore/DataBuff/Kubernetes/adml.yaml | 12 + .../DataBuff/Kubernetes/gwdataln.yaml | 14 + .../Datastore/DataBuff/Kubernetes/state.yaml | 66 +++ .../GlitchFlowAPIcomp/Dockerfile | 26 + .../GlitchFlowAPIcomp/app/Appfconf.py | 25 + .../GlitchFlowAPIcomp/app/__init__.py | 0 .../GlitchFlowAPIcomp/app/main.py | 129 +++++ .../GlitchFlowAPIcomp/envsetup.sh | 28 + .../GlitchFlowAPIcomp/requirements.txt | 3 + .../GlitchFlowAPI/kubernetes/deplapi.yaml | 38 ++ Release/LICENSE | 21 + Release/README.md | 479 ++++++++++++++++++ .../dockimg/gwpyimg/0.1.rc/Dockerfile | 39 ++ .../dockimg/gwpyimg/0.1.test/Dockerfile | 22 + .../gwclient/dockimg/gwpyimg/0.1/Dockerfile | 22 + Release/gwclient/dockimg/pytools/Dockerfile | 19 + .../gwclient/dockimg/scibase/0.1/Dockerfile | 22 + .../gwclient/dockimg/scibase/0.2/Dockerfile | 40 ++ .../gwclient/dockimg/scibase/0.3/Dockerfile | 22 + Release/gwclient/envsetup.sh | 18 + Release/gwclient/gwclient.yaml | 10 + Release/gwclient/lastrd.txt | 1 + Release/gwclient/main.py | 114 +++++ Release/gwclient/requirements.txt | 4 + 90 files changed, 3611 insertions(+) create mode 100644 Release/Client/client.yaml create mode 100644 Release/Client/client.zip create mode 100644 Release/DAG create mode 100644 Release/Datastore/DSLogic/DSlogcomp/.dockerignore create mode 100644 Release/Datastore/DSLogic/DSlogcomp/Dockerfile create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/DBCalls.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/DBhooks.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/DSDesc.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/__init__.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/__pycache__/DBCalls.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/__pycache__/DBhooks.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/__pycache__/DSDesc.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/__pycache__/__init__.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/__pycache__/config.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/config.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/__index__.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/DBhooks.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/DSDesc.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/cleanview.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/descview.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/dlogview.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/dumpfview.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/errors.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/freezeview.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/indexview.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/main.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/statview.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/streamview.cpython-38.pyc create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/cleanview.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/descview.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/dlogview.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/dumpfview.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/errors.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/freezeview.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/indexview.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/main.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/statview.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/main/streamview.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/app/templates/index.html create mode 100755 Release/Datastore/DSLogic/DSlogcomp/envsetup.sh create mode 100644 Release/Datastore/DSLogic/DSlogcomp/gunicorn.conf.py create mode 100644 Release/Datastore/DSLogic/DSlogcomp/requirements.txt create mode 100644 Release/Datastore/DSLogic/kubernetes/depl.yaml create mode 100644 Release/Datastore/DataBuff/Databuffcomp/.dockerignore create mode 100644 Release/Datastore/DataBuff/Databuffcomp/Dockerfile create mode 100644 Release/Datastore/DataBuff/Databuffcomp/class/appmodel/appmodel.go create mode 100644 Release/Datastore/DataBuff/Databuffcomp/class/appstr/appstr.go create mode 100644 Release/Datastore/DataBuff/Databuffcomp/class/dsstat/dsstat.go create mode 100644 Release/Datastore/DataBuff/Databuffcomp/class/envdef/envdef.go create mode 100644 Release/Datastore/DataBuff/Databuffcomp/class/fbufstat/fbufstat.go create mode 100644 Release/Datastore/DataBuff/Databuffcomp/class/fwrite/fwrite.go create mode 100644 Release/Datastore/DataBuff/Databuffcomp/class/go.mod create mode 100644 Release/Datastore/DataBuff/Databuffcomp/class/go.sum create mode 100644 Release/Datastore/DataBuff/Databuffcomp/class/neterr/neterr.go create mode 100644 Release/Datastore/DataBuff/Databuffcomp/class/netutil/netutil.go create mode 100644 Release/Datastore/DataBuff/Databuffcomp/class/simplelogger/simplelogger.go create mode 100644 Release/Datastore/DataBuff/Databuffcomp/main/adm/Dsstat.json create mode 100644 Release/Datastore/DataBuff/Databuffcomp/main/adm/StatDesc.json create mode 100755 Release/Datastore/DataBuff/Databuffcomp/main/envsetup.sh create mode 100644 Release/Datastore/DataBuff/Databuffcomp/main/go.mod create mode 100644 Release/Datastore/DataBuff/Databuffcomp/main/go.sum create mode 100644 Release/Datastore/DataBuff/Databuffcomp/main/log/LogStream.json create mode 100644 Release/Datastore/DataBuff/Databuffcomp/main/main.go create mode 100644 Release/Datastore/DataBuff/Databuffcomp/main/static/Dsstat.json create mode 100644 Release/Datastore/DataBuff/Databuffcomp/main/static/LogStream.json create mode 100644 Release/Datastore/DataBuff/Databuffcomp/main/static/StatDesc.json create mode 100644 Release/Datastore/DataBuff/Kubernetes/adml.yaml create mode 100644 Release/Datastore/DataBuff/Kubernetes/gwdataln.yaml create mode 100644 Release/Datastore/DataBuff/Kubernetes/state.yaml create mode 100644 Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/Dockerfile create mode 100644 Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/app/Appfconf.py create mode 100644 Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/app/__init__.py create mode 100644 Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/app/main.py create mode 100755 Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/envsetup.sh create mode 100644 Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/requirements.txt create mode 100644 Release/Datastore/GlitchFlowAPI/kubernetes/deplapi.yaml create mode 100644 Release/LICENSE create mode 100644 Release/README.md create mode 100644 Release/gwclient/dockimg/gwpyimg/0.1.rc/Dockerfile create mode 100644 Release/gwclient/dockimg/gwpyimg/0.1.test/Dockerfile create mode 100644 Release/gwclient/dockimg/gwpyimg/0.1/Dockerfile create mode 100644 Release/gwclient/dockimg/pytools/Dockerfile create mode 100644 Release/gwclient/dockimg/scibase/0.1/Dockerfile create mode 100644 Release/gwclient/dockimg/scibase/0.2/Dockerfile create mode 100644 Release/gwclient/dockimg/scibase/0.3/Dockerfile create mode 100755 Release/gwclient/envsetup.sh create mode 100644 Release/gwclient/gwclient.yaml create mode 100644 Release/gwclient/lastrd.txt create mode 100644 Release/gwclient/main.py create mode 100644 Release/gwclient/requirements.txt diff --git a/Release/Client/client.yaml b/Release/Client/client.yaml new file mode 100644 index 0000000..43f3c24 --- /dev/null +++ b/Release/Client/client.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + name: client +spec: + containers: + - name: client + image: alpine:latest + command: + - /bin/sh + - "-c" + - "sleep infinity" + diff --git a/Release/Client/client.zip b/Release/Client/client.zip new file mode 100644 index 0000000000000000000000000000000000000000..4dbeb155b562e8a8c8d54ce1593850f5b7fbf6c7 GIT binary patch literal 2890 zcmbVO2|Sc(82=nYj4)xdg+y*9*E(}DMzVmy&Iey>wfBygHdEfst))s8++z?}R+fcDHALcIX zkO<`Ii`#B*#|5!@#>F1Mg9r{qLM-fzJ7zA{D?W{!dwSnNUoRYy2@<+Ih#){*H$$N# zyZVkS_-4)mLHux#;Gly9m3cZ0N{i?yD3PnWW##0q-hX(n)5@}|vtv8D>YB2$GG#kj z&K%PTO@+hm(oI|Z)eY*3QU7%}6D(<;Bb ze185|8pjy))xJ_gq0-6xDn7}9P1^9=&VgPlz9URgGo93}rfYx_Ldp;%3KxYY5QDrt z0`_|lJ%BVcA>`0%H67{eHg7ZJ=Nv5!%}i}-{#z`(k5nOkGoyf0PY_WNnt{0%;CKqt zVL4q#Q}~S2j}0u)k9JP6Zm`>dO`!9g>Fh$c{o01>)icXdh{RRrr{rn zXZpR`ZJEY1MhZe&p`Jcm-`4R++45M8hz`1%yy2$&p$CEEQ1q6PP(@DvqLHXdU5h8q zN>%kLP9N=nE(jV;tMj469aji$a=Lx%?7ErXAc%g}Jo#|;&XaD}r|9?Sbj|!5UnXqc zyJ(!%mYThz`PmPSc>7DnWaCVE%*1rD`rb*?aQTdy574bXS{fz z=5~~zt06$m(b>vp;%!h{u4K`qFn-0_@KgP+XpmhHG-M;Z8ALAv(K|#1?MDd2&9`A=;*wn!dDnuw)sUZ^@-TLiOsn%RdD(5o!BM|wj9+S^wBxc4N zEUy03m?M8;hpcCcY}_PTqPznE6qVDB|AW->}e#6q-WeP zic;vx3-z)JyGK@Q!UbhCN@(Z5@n?;7eGltOW@{QmWR!WSAnqlPL^c}56I?iL!#$jV3(3d3<}M=@E5*ix~zl;Vqf z00Sowiw-V<=s_fWKe(TOQ$X=cN{VPP^I z1GV;KtBfnwe5)RKYfN^X%B=7&Q)B6E)ru(T&vc@GK!nNh8x)2i#Nu<)g(Z^=-BTuW8kn+JKo3Cj{{=sS#nmNdLpfFlLHoL%_?_%Y5v5N(%Rw6s7JGG6K7c$zaHi zioGZ$hD!nr@|V_!806tQZ)_Gxu?O~oSmdUdt~q1NtJK1~qi8-@mwxr0mQ0C&o;yZ^ z6yX;)HfuiAY?vUYr@W&*k%(W{C_zJt@D-~mcHP{oYelnARg>nu6F=$OCwVTq5ns+h z7^T&;DPynbqiA|1F4F1k*=|xgqZi$|&)Z5z_-6^y6iRB}DLK3*SxwZEj}85Hd|YOW zR1;GZqpI!o6OSW-p7S7t;Z=p%+I)GZ5|$D6CvU4VPXblX%A=jqxzi=<1on}kJlo}* zY|qm<8we2*thTj9R;KiDFVmF6q;R!cb*3Mp&y}9|)Q~H?b#xo4q1`Z$#**Lt`-y#a zTqB2K9^kb!iE^%9iAN@&r}`$tB_;V?&IP%RS~g}q1OBO^gY3G)Wn-70O}GEXVr{{~ zD#o_b$Q1^%7>geoF=D*W4q(LW0AAFe)sT>&IJ1i%`f9{Dstlz0VZ0jPN~3rdkTFt7 zF>!Ffxxu_P>e6w`L>-BlfTJ>pF92o?-(~eP=|ZrP_gz1OE*Y?1pntgmxHfmj?t>dS zlRGKKkqwptZtMRI9B!%$!DYdptp)sNEOu?or54G=9ErIC$7C8N0A?8G8pVdY8xxom zvm3FPEbzLu*)BCKCg@1aT{tMyz5p=8zSdw1ca4R>g&Z)pi#%i{+6eFz1R=qz4fGuk I93F!H1=MAXBUFSZ): + flag=True + else: + flag=False + + + return flag + + +### DAG DEFS +IniTrain = DummyOperator(task_id="start_training", dag=dag) + +sign_train = SimpleHttpOperator( + task_id="send_frz_sign", + method="POST", + http_conn_id="testapp", + endpoint="train", + data=json.dumps({"user":"airflow","token":"airflow"}), + headers={"Content-Type": "application/json"}, + + + retries=3, + retry_delay=timedelta(minutes=5), + dag=dag, +) + + + +chech_train_resp=BranchPythonOperator( + task_id="check_frz_sign", + python_callable=pick_branch, + dag=dag +) + + +###SENSOR BRANCH +next_sens = HttpSensor(task_id="next_sensor", + http_conn_id="testapp", + endpoint="stats", + response_check=lambda response: check_response(response), + poke_interval=10, + timeout=3600, + mode="reschedule", + dag=dag +) + +freeze= SimpleHttpOperator( + task_id="freeze_ds", + method="POST", + http_conn_id="testapp", + endpoint="train", + data=json.dumps({"user":"airflow","token":"airflow"}), + headers={"Content-Type": "application/json"}, + + + retries=3, + retry_delay=timedelta(minutes=5), + dag=dag, +) +####################################### + + +###BEST CASE BRANCH +next_metrics = DummyOperator(task_id="next_metrics", dag=dag) +########################################################## + + + +join_branch = DummyOperator(task_id="join_brc",trigger_rule="none_failed", dag=dag) + + +preproc =KubernetesPodOperator( + # unique id of the task within the DAG + task_id="preQ", + # the Docker image to launch + image="romanoa77/preq:0.4.air", + # launch the Pod on the same cluster as Airflow is running on + in_cluster=True, + # launch the Pod in the same namespace as Airflow is running in + namespace="glitchflow", + # Pod configuration + # name the Pod + name="airflow_preprocessor", + + # attach labels to the Pod, can be used for grouping + labels={"app": "preq", "backend": "airflow"}, + # reattach to worker instead of creating a new Pod on worker failure + reattach_on_restart=True, + # delete Pod after the task is finished + is_delete_operator_pod=True, + # get log stdout of the container as task logs + get_logs=True, + # log events in case of Pod failure + log_events_on_failure=True, + # enable xcom + do_xcom_push=True, + volumes = [k8s.V1Volume( + name="gwdataln", + persistent_volume_claim=k8s.V1PersistentVolumeClaimVolumeSource(claim_name="gwdataln"), + )], + volume_mounts=[k8s.V1VolumeMount(mount_path="/app/data", name="gwdataln", sub_path=None, read_only=False) + ], + dag=dag, + + #env_vars={"NAME_TO_GREET": f"{name}"}, + ) + + + + + +IniTrain>>sign_train>>chech_train_resp>>[next_sens,next_metrics] +next_sens>>freeze +[next_metrics,freeze]>>join_branch +join_branch>>preproc diff --git a/Release/Datastore/DSLogic/DSlogcomp/.dockerignore b/Release/Datastore/DSLogic/DSlogcomp/.dockerignore new file mode 100644 index 0000000..ffa1de1 --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/.dockerignore @@ -0,0 +1,4 @@ + +envsetup.sh + + diff --git a/Release/Datastore/DSLogic/DSlogcomp/Dockerfile b/Release/Datastore/DSLogic/DSlogcomp/Dockerfile new file mode 100644 index 0000000..ce8701a --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/Dockerfile @@ -0,0 +1,42 @@ +# syntax=docker/dockerfile:1 +FROM python:3.8.10-alpine AS comp + +RUN python -m venv /opt/venv +ENV PATH=/opt/venv/bin:$PATH + +COPY requirements.txt . +RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements.txt + +FROM python:3.8.10-alpine + +### Env var and cmd line arguments#### +ARG srvd=/var/www + +ENV ENDP_STAT=stat +ENV ENDP_DUMP=dumpLogF +ENV ENDP_DUMPF=dumpF +ENV ENDP_SEND=sendF +ENV DB_BURL=http://databuff-0.databuff.glitchflow.svc.cluster.local:8080/ +ENV ENDP_DESC=dstat +ENV ENDP_UPD_DESC=upddsc +ENV ENDP_CLEAN=cleanall +ENV MAX_SIZE=2000 + + + + +ENV GREET="Hello from Gunicorn server!" +ENV PATH=/opt/venv/bin:$PATH +###################################### + +RUN mkdir ${srvd} + +COPY --from=comp /opt/venv /opt/venv +COPY gunicorn.conf.py /etc +COPY app ${srvd}/app + +WORKDIR ${srvd} + + + +CMD [ "gunicorn", "-c","/etc/gunicorn.conf.py", "app:create_app('default')" ] diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/DBCalls.py b/Release/Datastore/DSLogic/DSlogcomp/app/DBCalls.py new file mode 100644 index 0000000..7270e62 --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/app/DBCalls.py @@ -0,0 +1,60 @@ +import requests +import json + + +""" +GIN CALLS + +router.GET("/stat", getStat(&StatDesc)) +router.GET("/dumpLogF", getLogF()) + +router.POST("/sendF", postFile(&StatDesc)) + +""" + +""" + +Response struct + +response={'code':req.status_code,'resp_msg':req.json()} +response={'code':req.status_code,'resp_msg':req.text} + +A data structure class could defined + + +""" + +class RqCalls: + + def __init__(self,url): + self.baseurl=url + #self.max_time=1 + self.RqHnd=requests.Session() + + def getReq(self,dest): + + + + req=self.RqHnd.get(self.baseurl+dest) + + + response={'code':req.status_code,'resp_msg':req.json()} + + return response + + + def postJson(self,dest,data): + #input=json.loads(data) + input=data + + + req=self.RqHnd.post(self.baseurl+dest,input) + + response={'code':req.status_code,'resp_msg':req.text} + + return response + + + + + diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/DBhooks.py b/Release/Datastore/DSLogic/DSlogcomp/app/DBhooks.py new file mode 100644 index 0000000..a18b35c --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/app/DBhooks.py @@ -0,0 +1,107 @@ +from . DBCalls import RqCalls + + +""" +GIN CALLS + +router.GET("/stat", getStat(&StatDesc)) +router.GET("/dumpLogF", getLogF()) + +router.POST("/sendF", postFile(&StatDesc)) + +""" + +""" + +Response struct + +response={'code':req.status_code,'resp_msg':req.json()} +response={'code':req.status_code,'resp_msg':req.text} + + +""" + +class DBhook: + """ + def __init__(self,burl,stat,dump,send): + self.Caller=RqCalls(burl) + + self.serviceurl=burl + self.status=stat + self.log_dump=dump + self.send_data=send + """ + def getBufStat(self): + + buf_desc=self.Caller.getReq(self.Urls['stat']) + + return buf_desc + + def getLogDump(self): + + log_cont=self.Caller.getReq(self.Urls['dump']) + + return log_cont + + def getDumpF(self): + + f_cont=self.Caller.getReq(self.Urls['dumpf']) + + return f_cont + + def getDSDsc(self): + + buf_desc=buf_desc=self.Caller.getReq(self.Urls['desc']) + + return buf_desc + + def postDataStream(self,data_struct): + + + #postJson expects a dict rapresenting a json object + post_resp=self.Caller.postJson(self.Urls['send'],data_struct) + + return post_resp + + def postDsc(self,data_struct): + + + #postJson expects a dict rapresenting a json object + post_resp=self.Caller.postJson(self.Urls['upddesc'],data_struct) + + return post_resp + + def postFlush(self,data_struct): + + + #postJson expects a dict rapresenting a json object + post_resp=self.Caller.postJson(self.Urls['flush'],data_struct) + + return post_resp + + + + + def connect(self,url_dict): + self.Urls=url_dict + self.Caller=RqCalls(self.Urls['burl']) + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/DSDesc.py b/Release/Datastore/DSLogic/DSlogcomp/app/DSDesc.py new file mode 100644 index 0000000..7eb8387 --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/app/DSDesc.py @@ -0,0 +1,34 @@ +class AppDesc: + """ + def __init__(self,size): + self.buf_state='OPERATIONAL' + self.max_buf_size=int(size) + """ + + def initDesc(self,size): + + self.max_buf_size=int(size) + self.frz_state='FROZEN' + self.op_state='OPERATIONAL' + + + + def isFreeze(self,curr_size): + + code=0 + + if(curr_size>=self.max_buf_size): + code=1 + + return code + + def isOp(self,status): + code=0 + + if(status==self.op_state): + code=1 + + return code + + + \ No newline at end of file diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/__init__.py b/Release/Datastore/DSLogic/DSlogcomp/app/__init__.py new file mode 100644 index 0000000..91fcddc --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/app/__init__.py @@ -0,0 +1,49 @@ +from flask import Flask, render_template + + +from . import DSDesc +from . import DBhooks + +from .config import config + +AppStatus=DSDesc.AppDesc() +DBConn=DBhooks.DBhook() + + + +def create_app(config_name): + + app=Flask(__name__) + app.config.from_object(config[config_name]) + #config[config_name].init_app(app) + + + + + AppStatus.initDesc(config[config_name].env_max_size) + + + DBConn.connect(config[config_name].env_dict) + + from app.main.main import main_bp as mainbp + main_blueprint=mainbp + app.register_blueprint(main_blueprint) + + + return app + + + + + + + + + + + + + + + + diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/__pycache__/DBCalls.cpython-38.pyc b/Release/Datastore/DSLogic/DSlogcomp/app/__pycache__/DBCalls.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f248fec82e0c8e89b40552ffed2016c5fc36c198 GIT binary patch literal 958 zcmZuw&2G~`5T5;UV){?11PF1d5Ry5>FMv=*EAgW$NR7DoVj1r?ByzmYt|LL^9uR#D z4$ZNzv{w#@SK!2qgCUhzX=Z0oPIffCjh?FVSS+wj!@hoDg}xa zkgRr=uvehC5?7$aSDtXiMi9xLh=XBHdVO&`9HF=;s5B%@0c9wD1&L5XN!+ChR1fz+ z^|4au`)BjxtSDSg$=xEG<|f?5#ANFS244vGfnwd!!za`gdb%QRgguvyV{PwuL+l!mZX92}|9vyB+9% S**<)}y7>j85|j|K@ZoQSnbAA| literal 0 HcmV?d00001 diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/__pycache__/DBhooks.cpython-38.pyc b/Release/Datastore/DSLogic/DSlogcomp/app/__pycache__/DBhooks.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..26b3f01084d15ebc296b9cbec61d5a87f615651b GIT binary patch literal 1837 zcmbtVJ8#=C5T;~Vk{@y2Y10CQ(J2NDYv~l!Mc^1I8U#TRBa0S75$VLOY$>EvpoTLg zKcpGGcj~X`thH1BLZ{x*b}XmKlG5Ov$UE}Bd+>a^v{WQ8j$Z!g{xk{sg^TRS!D0`l zdb}n5H%OgDf}H& zuN5;*F0UWOpF9Ij}nm$~eC^hEtCp(Gp6M49|ri%hF*T&HL;UHG-Oh^d? z3Jm~m!+{-GKZs^3etIG~Ai;N$DI`mol74FeM6-n}tsa+LM_f8Lk{4{yNz z`1B5$W@^J(Xr{vBS~;~T9e90CX^%?SB{#wIynyaG16Al?!|1ic2n zVkr^qltfMr{w;zk?TrB(BuS|tDHAkx1QX6RNnPQJ4(N{Xgb&&i0n`btKdP#+H2LzP zy9XrXbdU35Ueufqj4Y>5O|g{G^CpN1=o_9brrgw7Ef4l59}Z7OAKy=s(eeIv(k%PN zn?xvZP|WK=Q>n?x5$zq}8Tk#cc>p&1Ma;T50!V;$L$nVKf4z%8tC@_mQW{gJxKz28m5OVf74w-A z@i?n9Q!6dw@#(QTFLKbOI?vUtinFQ$3s*R-E^UCTLnFXxJP?F+sYjPP>pc(e61H}L z?|r|6M_>s0ovmQ{FXW=5cLf6r-u=c&Zn5Mo1rbq=*CI!EK&4^6(3;~^*3FfW&19`W zEP%`$X(^X-2P}gw!d$@SvJGtOl>f%Y{$SJiqXCnPs>NhI{3Hvb`{4TD})M!b#-Yfb)4albqWw=kOSd63;!(zb>+J9q~CAD(75>*yte&oN}WjKC{N5 iHLec1X&X@23xXhQya}Mk$iJ@H;6>Z}C1(5tCcgm%AJWVK literal 0 HcmV?d00001 diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/__pycache__/__init__.cpython-38.pyc b/Release/Datastore/DSLogic/DSlogcomp/app/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7b076ed65812661ffcd4fecd455aca70103f3a9c GIT binary patch literal 756 zcmYjPOK#gR5amZd+j3%b*CTinU7#qM#OR_Z(gH@i;D(Sz+L0rQ1j!U_Ag|g*kD)Ak zgkDA4E{a^Cs}AJ^2m!wNaYi$5^l3hyA~-+){@#8d2>l7h^8-Hg2Cn-EAdtWfQkY>y z7{N#oK^v+f7L9rYdd%WckDEj#EWt=deR_{rCgXkraw-y${y=OdvNc*x9-t~(k|BQE zlN-rO92;(7Htyj`jprxyeYpriwa8|nA*1Iaac{&j&;aU<)irGgY$#jMiSmz5tT=f2Dp&>bCY_CYHb= z?&>=UlU`ivwjEL!u7w;R4#J`aZC)C47QQxu%|iJXR_Y$+6p~x4)1*finNz71vtHqHLsdwJsW6S!s0PEo3PzidE@L=e3o^YW-gC z>I!6|ca>JAfbs?BVI0oS%(r0*xv)A8{!rU301(OWB3aM{$#I0I)CT9(GH3BN?0sa; zQt%DP#vp7GU?ea-Hzx38!+*+vp0Iv93-M57?Ee{@f^W!PLDRR5d>wY=a)1TRaZYk* HBa8k6IHJ5f literal 0 HcmV?d00001 diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/__pycache__/config.cpython-38.pyc b/Release/Datastore/DSLogic/DSlogcomp/app/__pycache__/config.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6bba74f9780b95ebecadffa1b291bae971a6bdf0 GIT binary patch literal 1160 zcmZuw&2H2%5VrHb*-ih%f!;Z=m*xc!YP;DMP`j$wu7rHC(%9P#$|k|iuAtZUgv1+g zphq5oSMe2b=p(?1v9l{yfTQ{H&Db7~zey%h6e1YSk8jfl8=>#o*&Hr7hhXv*07DEH zh#-#M;xCp#n>+XT9uxNzF^k!Eh}pLm@xXJK3!clpTbuaYXBf=;Y?FZDG3xsdkPnSA zjhjPPa|k9s0&qkyL&W0tTZiF&L|nrxV4h(%FyAl-SYVh7tYer5EHumq)-@~u)-$XF zEaF?d%UvGu4iCAyWUMmcOTua&ED3L#!H2yvMye^E0-+-b3pvkZf3>!G=}0hn|E7=nx7(N_Rvr*$p5 z5H#@XMLetKJWdNPWnRTal?h%~aU;^aoL4MP(k7KnC3u{S&#LP@gLhF~XVtupAyb?g zZ4T-WD#**cp%8nbkF@}hb+g@Scd(ut_4iajX_?MBr7ENp&R7(h_b7e0NQ>2sRY~RZ za+!;&g!Yz8s^w^zzBaTHVxR#vo>`$qf)4=VMy}O1kgR1!RkLlLzNa$3;q79hAE=sV zjcW5~*3;+Z*_=1CiixeYUVoGmk&fwWZiC6E009pFek=!Do6k=#w6NLr&kvwGcBoVB z0IYuihKbEA=$@@Q317Bqo{ajA(5DK$RO|vMHyIvJPB+RkEjBeihIw6{=}CZv_936m z_)377QPUh5UGgE$gC0oLBq5GmcH_c7A{Hs)Q2%D>zjLK_2h5wbH Vf@f`?pX*D?03g7RqOJWM?>E^E8{Pl_ literal 0 HcmV?d00001 diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/config.py b/Release/Datastore/DSLogic/DSlogcomp/app/config.py new file mode 100644 index 0000000..0520703 --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/app/config.py @@ -0,0 +1,50 @@ +import os +basedir=os.path.abspath(os.path.dirname(__file__)) + +""" + +ENV. TO BE DEFINED INSIDE THE CONTAINER IMG + +ENDP_STAT +ENDP_DUMP +ENDP_SEND +DB_BURL +ENDP_DESC +ENDP_UPD_DESC +""" +class Config: + env_stat=os.environ['ENDP_STAT'] + env_desc=os.environ['ENDP_DESC'] + env_upd_desc=os.environ['ENDP_UPD_DESC'] + env_dump=os.environ['ENDP_DUMP'] + env_dumpf=os.environ['ENDP_DUMPF'] + env_send=os.environ['ENDP_SEND'] + env_burl=os.environ['DB_BURL'] + env_max_size=os.environ['MAX_SIZE'] + env_flush=os.environ['ENDP_CLEAN'] + + env_dict=dict(burl=env_burl, + stat=env_stat, + dump=env_dump, + dumpf=env_dumpf, + send=env_send, + desc=env_desc, + upddesc=env_upd_desc, + flush=env_flush + ) + + @staticmethod + def init_app(app): + pass + +class DevConfig(Config): + DEBUG=False + +#class ProdConfig(Config): + + +config={'dev':DevConfig,'prod':None,'default':DevConfig} + + + + diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/main/__index__.py b/Release/Datastore/DSLogic/DSlogcomp/app/main/__index__.py new file mode 100644 index 0000000..e69de29 diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/DBhooks.cpython-38.pyc b/Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/DBhooks.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d48daaf450e27d7b818981f542a8e0a57abffd82 GIT binary patch literal 1080 zcmbtT&2G~`5Z+xoj$_)i{kU09f7^uo#Lc!@usMREUw{)t z(3E5}qLjQRf(h%A2IC=O#$a;p^8kh5VJ})_cj=7kkP>(qjztvs~9BeQN1<=MY4=j@r;UO$)az?&G`e*Elo`Vn} zQBYdqTMG4RoS7_7;ZUKO+!fY73?(zBCO9=2Wv{V z_y0pm?L%1A1^1T`ik}pDc^{*X#cTw1c?0-+8LjUVlxAZn5wbTjJ%+`tOf|h?=g=#6 zg^Juoq00}t$dwClkk3f<0Na4;QdLPA^$^09)Qf*<_VH!}(6Cf8&IS;nrE2f9)swY& zcQIqXi*1m{ndH3jIL``ErMTYVd|Ab5d!zOsmO4O()mD$tVbg6ntewJ)pTW>b-({}r oI<8|*$G+8cM?>?In*YF@ZoN0*Oar)YTNdxIB zxUWUTKnz&mBzV5I&`bGZ4+X^{4lEMDLENy&0S8>zQ~0r`cui zU@*$gl59NfjpKBJDq-wSzZIU8=_JjiBz(q2!g!s}x9`&ER_270>28s{%7?m9=0jNo zB8=)sC;jL!TyjnRkTsB5cqO41g+J6v&DERPu#T!BcZ|<>s<vf!=rF3>!Wa#tL+=XT(EP>k%#{dH9RvRi@_a7};$#}+s8lbP0%2BC114Af@&UTThPy`SJC;~`_-q5n)jCvqRy mN;H0&YDn8qn>_Bho?}1XT0$%@S^xI5@b*d+ujs}v-0Tl`gWAmi literal 0 HcmV?d00001 diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/cleanview.cpython-38.pyc b/Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/cleanview.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..93c0f9e70ae9ab759ab965318bb3d49b33116207 GIT binary patch literal 641 zcmYjPJ&zMH5cSu7g(Nx&khqR^o7)CDgitO@d4~uz+nhFQC)s4ZUTiOLQ6dViz5f6u z|ALmk;Fc;q6&2$>opdA3(>!bZ{AT12qtO6CMANU!Ul^f3o;(Z&*o zBssyJlbTAFGpH$Nbs)n$#7MAB!~-6#Fpq8tk9l%~zTrIPgE>kc9YG1D#6?zY zIi#5ZShT?hWXX5BM?dfZ?=i=x;CoB-3(IT(oSuR^PB`PiEj^HCY#9%EbkFw$T*nuV zm5PhwZ_sqH2FF*@tac|w+tyWC*h)3o#;67c3ym#oXIv~RDeQGCcF9Dmt!SRl=G9-= zavEwso@2bQg${k0w943-A5r@e95q&_RF8cfs?N5ZbrHPfAO4B-V>ti!GFvMtvZ5Bo zR4Ow?!x!pHHl5dMRVvwL(4YBzW@Rmk=2Im;pS9O6?l(Em7fFq7eRN*nkh!ajr zQqeKRNZ=cfd)&`44_*ildGw5)@Yv^DBNVq*phPjYVU-rM$yL1~>pe)02XZ!h3>nv4 z=*-g7lZ%EY%v5rnIo*Fbjq3rl-UBe`6~Cb+T7ah>xVa^27iQW+OT55v*Be?dlU%Gm zfo&rtbUmqb9uws<*(et#86`T4i4El3)N^CQw5}cR9jOj*e$UxdR)VFa(7KSUl$jE> zWJW=dN^+K@Ce=nN!IIIL%!>@(Qs$YgY6f317nr$dcZIm`*AF&uM|I}I=Q;vFB%*B` xQMC>8Fb-@7WPK-VK)J*gtquDMT#AO-xZ8lPqeTZ`XBb5S_hm2!}O^h3z$LLurf=)W*<&4GlTi|^R9gbLosJ8WQ$t~%)F)+rCMb~~^kYl54-q4DOD@$DC~ zO;mENrb1_lQ`)OcGvn2j%m%nTaZDiVO4XGKQ>85P6UrD_KRvpXg3~nTTCXJ4X~Aal zfeuFKcP@pZ$fx!)%JAiG#qYc7(S$hr0(&1v=m-J4C}@!=Di5$8CV`1j*R{GGl=d<@5L%?=w#hHBb8IaD literal 0 HcmV?d00001 diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/dumpfview.cpython-38.pyc b/Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/dumpfview.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0c140e37c9ec4e1ab99b92c02267c6c7d7a530cb GIT binary patch literal 531 zcmYjOv2GMG5FPLOyi0T!>Z_=?i76-}gvb$^C_)M}+eDikXS2axTec4bN-Aji2B72@ z_yWGcE!F)(s*LXxDI?9ZXY^*~js0P<*aNa>%fFXD8NmM>JT^|jEAntifPhScVJ%~T z1(v!nWn0o6kZay@#ja;KLoqO{eiT^alkf12wF3DWGBi~&!flFOELN|N z>@=Nz6%yCRU8eHwoA+CuidPmxpX(kh*JM8;4=)HJ{9?c17H&w}l2m;x{BydOOSok> z3_0Z>=a3IDvYaC=v(v21NS8fntH5*pzdK#0K67+R&(;oKc%>-(t=74a_JU)evt$_(fm5tDa}0T eRa^R2=Oft9;=UjS^kG!k=#kYHiyQn7S7<(fb%C6|!&Bw$1EAu#eQe8R0v z-I7>299z{G{T42rSMK=>$nL4VF2W126-xT=O^3LKiRW z8RkOo%vd$pppI4051m@IbrUvUz}N!u8?jJjnZ{A zcb!RVGE5wa@i;`DRz+W8>zGiJWdLNZO&_a9uxuig!)a$qap%7pdt8i5j#HuKPeK7Sa6oom?m>f(<_he(7_LM ziHjXNoT6;Bg;!8UOeERYo+)kO4@`BURm==|dCrdIZJ1#b@v?4@XWbUtJJ`fZm-<`H zzHdQ@O(d_oZVI_xK@d(34n^B8lf06QEg4nDfB$a!(KTZFR8VzdAOCMwcmU@Rj(Q8A z(KX(nHCn+CcF`JZ@0zUe2Frczp9O1Ahu6`{+hB}z3^m@RwZEfxt4=>_X>V1F|#E zgjQ~^Ow4P}^rB-~DkCT}>740PX7_uDI+vJN}9^kc}ne_LKl!OI$cWI9>)ve8`0(rUONAz8Vs zZEdX7yj-r{$aU9%@8!CYLY0uJ^fy_CW_;-Q$A$XrNMQcKnP*^7oZ~#olUW?$2u<*D z?$bg)h3}X10+jZ(*#rsh1kx^i+F-PQn+|CL?$8IKK@1N^gMiI2@+K;2`{Ut(a! L*D=FkraAr#-S@^E literal 0 HcmV?d00001 diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/main.cpython-38.pyc b/Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/main.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f83d2f47a03c1e25260871225f706646f843e55 GIT binary patch literal 436 zcmYk2J5B>J5Qgnd_L(da$4G-OfDl3v4TvV9akDb(*|5U1w(JeBmdn5uxCw5las?{v z4Fx0ln}5bH{c~H_6$$Z`e|GO7f5+l~%tCP-#_TeqEqQ1OPJ218%MxUeThvv@gY1}G( z*$)z)AA5N68Ca{W^MxD5ICToC^mbB}8!ZSt=*|MX!lcVqff)V{gKG~>xkJ-Md9yn5 zX{UsHuu1sO7fmw=gr>34uO~JQ$z~CGL=jO&#A(?@#4@4|_#pN;>WH5!cCQhbP~cpz znbNHVquFSKJ=Bt|#VFjUEwJ_GT5tO{Y5S_ViLoR*5!uFbMlTt*Ih+ROUAZfxSySYT~ZjakLzc7`y{s(OY|B1h0 zTdVv9D`zhV8JNe;@*eZv?o}AJfa?09UVb9Lj~VvMq2ZKT^$9p|Od-Q5B5;e_sg>DN z8$sZW!yWD>$h|j<`#gAs7o58MU;DNN)t-A5>XB6<%AYTmiA;_6?LWkxF`(s3(AmKzkRlnnP71$lv+!s;+)Ur z0~<|JnJi>hFyb>4jF|*?YjNKz9!vA4F0~2>zzMuI27YrpjyxTbre`z_lqQviohi2@ eAEIG4`}d$@Xu)wNd7X+A`YTW#K^xn42mb)d<7ndm literal 0 HcmV?d00001 diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/streamview.cpython-38.pyc b/Release/Datastore/DSLogic/DSlogcomp/app/main/__pycache__/streamview.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0c8da9cdec378028d795b5f7b4d71e0ac619bf6d GIT binary patch literal 762 zcmYjP&2H2%5VjpBPQrEt2q7(8xNZ-d8z+Q-?zycRxa3mOwfCn@Vq-hdu6D(tT>B2) zBPX7NSMZh7J^>OZ#@SY?Bah~Dz8TLq<4=2gDT4KD|NC@5KiiIVRxpeNJ&$?B*$ngehVX=^KhQNU68FM zq9Jiii!ZG(*2O2nDA^b>#E#YFqB%WNTjELBCGVBWZE0HrAR2!-kWI5>nXy`wwWG&J z`Nu9w*tz6p;#1$R5FEb`U>gp`qFcN{YqSDM1CTL5Yi!AFu)-UxPb@tT*917xifr&Y zSOq))7QG<)vt>303{lsZQy%i@CRo#^Zy9GizQ-#H@{&UrYtVCAo0((WbhSG=kz8zl zLlMi=ic6cR3UopfqB}xdFKS`0lvq9)m$o!kYVm5SnmxjPmy;m(lB7;0_fK;1copZ7y2uOe(HZjvbCzUj+ILJs(&Iq zfb)N^v$?EAR#w88h0ILZ@H2Utjq^&*CbCu;@H0QEyGeXqh;JixcVMNnu|`ruGBrqJo1@4qy1ZSsMEUBA8IIBbn#uO c56lbCjo;Lgx0QGcJwwJn^vAg8zp#)017Xs`5dZ)H literal 0 HcmV?d00001 diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/main/cleanview.py b/Release/Datastore/DSLogic/DSlogcomp/app/main/cleanview.py new file mode 100644 index 0000000..f13eaed --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/app/main/cleanview.py @@ -0,0 +1,22 @@ +from flask import request,Response +from app.main.main import main_bp +from .. import DBConn + +@main_bp.route('/flushbuf',methods=['POST']) +def flushBuf(): + + data=request.data + + + + output=DBConn.postFlush(data) + + if (output['code']==201): + appresp=Response(str(output['resp_msg']),status=201,mimetype='application/json') + + else: + appresp=Response("Broken FS",status=500,mimetype='application/json') + + return appresp + + diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/main/descview.py b/Release/Datastore/DSLogic/DSlogcomp/app/main/descview.py new file mode 100644 index 0000000..9d58ffc --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/app/main/descview.py @@ -0,0 +1,14 @@ +from app.main.main import main_bp +from .. import DBConn + + +@main_bp.route('/dsdesc',methods=['GET']) +def getDsc(): + output=DBConn.getDSDsc() + + + if (output['code']==200): + appresp=output['resp_msg'] + + + return appresp \ No newline at end of file diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/main/dlogview.py b/Release/Datastore/DSLogic/DSlogcomp/app/main/dlogview.py new file mode 100644 index 0000000..3269add --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/app/main/dlogview.py @@ -0,0 +1,24 @@ +from app.main.main import main_bp +from .. import DBConn + + +@main_bp.route('/dumpflog',methods=['GET']) +def getFlog(): + + + output=DBConn.getLogDump() + + #print(output['resp_msg'],file=sys.stderr) + + + + if (output['code']==200): + appresp=output['resp_msg'] + + """ + else: + appresp=Response("Internal error!",status=500,mimetype='text/plain') + """ + + + return appresp \ No newline at end of file diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/main/dumpfview.py b/Release/Datastore/DSLogic/DSlogcomp/app/main/dumpfview.py new file mode 100644 index 0000000..0794944 --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/app/main/dumpfview.py @@ -0,0 +1,29 @@ +from app.main.main import main_bp +from .. import DBConn +import json + + +@main_bp.route('/dumpf',methods=['GET']) +def getFls(): + + + output=DBConn.getDumpF() + + #print(output['resp_msg'],file=sys.stderr) + + + + if (output['code']==200): + appresp=output['resp_msg'] + + + + if(appresp ['nmlist']==None): + appresp={"nmlist":[]} + + + + + + + return appresp \ No newline at end of file diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/main/errors.py b/Release/Datastore/DSLogic/DSlogcomp/app/main/errors.py new file mode 100644 index 0000000..bcdcf1b --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/app/main/errors.py @@ -0,0 +1,6 @@ +from app.main.main import main_bp + + +@main_bp.app_errorhandler(500) +def handle_bad_request(err): + return 'Internal error!', 500 \ No newline at end of file diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/main/freezeview.py b/Release/Datastore/DSLogic/DSlogcomp/app/main/freezeview.py new file mode 100644 index 0000000..50ef77e --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/app/main/freezeview.py @@ -0,0 +1,26 @@ +from flask import request,Response,make_response +from app.main.main import main_bp +from . import statview +from .. import AppStatus +from .. import DBConn + + +@main_bp.route('/freezeds',methods=['POST']) +def freezeApp(): + + buffer=statview.getBufStat() + + + read_size=buffer['buff_size'] + + if(AppStatus.isFreeze(read_size)): + data=request.data + DBConn.postDsc(data) + appresp=make_response(buffer,'201') + + + else: + appresp=Response("Waiting for more incoming data.",status=200,mimetype='text/plain') + + + return appresp \ No newline at end of file diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/main/indexview.py b/Release/Datastore/DSLogic/DSlogcomp/app/main/indexview.py new file mode 100644 index 0000000..c4922c9 --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/app/main/indexview.py @@ -0,0 +1,32 @@ +from flask import render_template +from app.main.main import main_bp +from .. import AppStatus +from .. import DBConn + + + + + +@main_bp.route('/') +def index(): + + output=DBConn.getBufStat() + output_desc=DBConn.getDSDsc() + + output_f=DBConn.getDumpF() + + #print(output['resp_msg'],file=sys.stderr) + + if (output['code']==200): + + + appresp=render_template( + 'index.html',size=output['resp_msg']['buff_size'] + ,state=output_desc['resp_msg']['dstatus'], + nitm=output['resp_msg']['n_itm'], + user=output_desc['resp_msg']['user'], + token=output_desc['resp_msg']['token'], + files=output_f['resp_msg']['nmlist']) + + + return appresp \ No newline at end of file diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/main/main.py b/Release/Datastore/DSLogic/DSlogcomp/app/main/main.py new file mode 100644 index 0000000..1aa51a1 --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/app/main/main.py @@ -0,0 +1,5 @@ +from flask import Blueprint + +main_bp=Blueprint('main_bp',__name__) + +from . import indexview ,errors,statview,dlogview,streamview,freezeview,cleanview,descview,dumpfview \ No newline at end of file diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/main/statview.py b/Release/Datastore/DSLogic/DSlogcomp/app/main/statview.py new file mode 100644 index 0000000..6137fdd --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/app/main/statview.py @@ -0,0 +1,23 @@ +from app.main.main import main_bp +from .. import DBConn + + +@main_bp.route('/bufstat',methods=['GET']) +def getBufStat(): + + + output=DBConn.getBufStat() + + #print(output['resp_msg'],file=sys.stderr) + + + + if (output['code']==200): + appresp=output['resp_msg'] + """ + else: + appresp=Response("Internal error!",status=500,mimetype='text/plain') + """ + + + return appresp \ No newline at end of file diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/main/streamview.py b/Release/Datastore/DSLogic/DSlogcomp/app/main/streamview.py new file mode 100644 index 0000000..cefbd27 --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/app/main/streamview.py @@ -0,0 +1,38 @@ +from flask import request,Response +from app.main.main import main_bp +from .. import AppStatus +from .. import DBConn + + + + +@main_bp.route('/stream',methods=['POST']) +def pushStream(): + + desc=DBConn.getDSDsc() + state=desc['resp_msg']['dstatus'] + + """""" + + if(AppStatus.isOp(state)): + + data=request.data + + #print('Incoming data',file=sys.stderr) + #print(request.data,file=sys.stderr) + + + + output=DBConn.postDataStream(data) + + if (output['code']==201): + appresp=Response(str(output['resp_msg']),status=201,mimetype='text/plain') + """ + else: + appresp=Response("Internal error!",status=500,mimetype='text/plain') + """ + else: + appresp=Response("Datastore frozen.",status=503,mimetype='text/plain') + + return appresp + diff --git a/Release/Datastore/DSLogic/DSlogcomp/app/templates/index.html b/Release/Datastore/DSLogic/DSlogcomp/app/templates/index.html new file mode 100644 index 0000000..229bfa9 --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/app/templates/index.html @@ -0,0 +1,52 @@ + + + +Datastore status + + + + + + + + +

Buffer size {{ size }} bytes

+

Datastore state {{ state }}

+

Number of stored files {{nitm}}

+
+
+ +

Operator

+ +

User: {{ user }}

+

Token: {{ token }}

+ +
+
+ +

Files

+ +
    + + {% for file in files %} + +
  1. {{ file }}
  2. + + {% endfor %} + +
+ + + + + + + diff --git a/Release/Datastore/DSLogic/DSlogcomp/envsetup.sh b/Release/Datastore/DSLogic/DSlogcomp/envsetup.sh new file mode 100755 index 0000000..5aeed17 --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/envsetup.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +###va lanciato con source + +< +2009-2015 (c) Paul J. Davis + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. +""" + +# +# Server socket +# +# bind - The socket to bind. +# +# A string of the form: 'HOST', 'HOST:PORT', 'unix:PATH'. +# An IP is a valid HOST. +# +# backlog - The number of pending connections. This refers +# to the number of clients that can be waiting to be +# served. Exceeding this number results in the client +# getting an error when attempting to connect. It should +# only affect servers under significant load. +# +# Must be a positive integer. Generally set in the 64-2048 +# range. +# + +#We can pass environment variables defined in the dockerfile +import os + + +bind = '0.0.0.0:5000' +backlog = 2048 + +# +# Worker processes +# +# workers - The number of worker processes that this server +# should keep alive for handling requests. +# +# A positive integer generally in the 2-4 x $(NUM_CORES) +# range. You'll want to vary this a bit to find the best +# for your particular application's work load. +# +# worker_class - The type of workers to use. The default +# sync class should handle most 'normal' types of work +# loads. You'll want to read +# http://docs.gunicorn.org/en/latest/design.html#choosing-a-worker-type +# for information on when you might want to choose one +# of the other worker classes. +# +# A string referring to a Python path to a subclass of +# gunicorn.workers.base.Worker. The default provided values +# can be seen at +# http://docs.gunicorn.org/en/latest/settings.html#worker-class +# +# worker_connections - For the eventlet and gevent worker classes +# this limits the maximum number of simultaneous clients that +# a single process can handle. +# +# A positive integer generally set to around 1000. +# +# timeout - If a worker does not notify the master process in this +# number of seconds it is killed and a new worker is spawned +# to replace it. +# +# Generally set to thirty seconds. Only set this noticeably +# higher if you're sure of the repercussions for sync workers. +# For the non sync workers it just means that the worker +# process is still communicating and is not tied to the length +# of time required to handle a single request. +# +# keepalive - The number of seconds to wait for the next request +# on a Keep-Alive HTTP connection. +# +# A positive integer. Generally set in the 1-5 seconds range. +# + +workers = 1 +worker_class = 'sync' +worker_connections = 1000 +timeout = 30 +keepalive = 2 + +# thread - number of threads per workers + +thread=1 + +# +# spew - Install a trace function that spews every line of Python +# that is executed when running the server. This is the +# nuclear option. +# +# True or False +# + +spew = False + +# +# Server mechanics +# +# daemon - Detach the main Gunicorn process from the controlling +# terminal with a standard fork/fork sequence. +# +# True or False +# +# raw_env - Pass environment variables to the execution environment. +# +# pidfile - The path to a pid file to write +# +# A path string or None to not write a pid file. +# +# user - Switch worker processes to run as this user. +# +# A valid user id (as an integer) or the name of a user that +# can be retrieved with a call to pwd.getpwnam(value) or None +# to not change the worker process user. +# +# group - Switch worker process to run as this group. +# +# A valid group id (as an integer) or the name of a user that +# can be retrieved with a call to pwd.getgrnam(value) or None +# to change the worker processes group. +# +# umask - A mask for file permissions written by Gunicorn. Note that +# this affects unix socket permissions. +# +# A valid value for the os.umask(mode) call or a string +# compatible with int(value, 0) (0 means Python guesses +# the base, so values like "0", "0xFF", "0022" are valid +# for decimal, hex, and octal representations) +# +# tmp_upload_dir - A directory to store temporary request data when +# requests are read. This will most likely be disappearing soon. +# +# A path to a directory where the process owner can write. Or +# None to signal that Python should choose one on its own. +# + +daemon = False + +####### ENV VAR################ +raw_env = [ + 'DJANGO_SECRET_KEY=something', + 'SPAM=eggs', +] +################################ +pidfile = None +umask = 0 +user = None +group = None +tmp_upload_dir = None + +# +# Logging +# +# logfile - The path to a log file to write to. +# +# A path string. "-" means log to stdout. +# +# loglevel - The granularity of log output +# +# A string of "debug", "info", "warning", "error", "critical" +# + +logfile='-' +errorlog = '-' +loglevel = 'info' +accesslog = '-' +access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' + +# +# Process naming +# +# proc_name - A base to use with setproctitle to change the way +# that Gunicorn processes are reported in the system process +# table. This affects things like 'ps' and 'top'. If you're +# going to be running more than one instance of Gunicorn you'll +# probably want to set a name to tell them apart. This requires +# that you install the setproctitle module. +# +# A string or None to choose a default of something like 'gunicorn'. +# + +proc_name = None + +# +# Server hooks +# +# post_fork - Called just after a worker has been forked. +# +# A callable that takes a server and worker instance +# as arguments. +# +# pre_fork - Called just prior to forking the worker subprocess. +# +# A callable that accepts the same arguments as after_fork +# +# pre_exec - Called just prior to forking off a secondary +# master process during things like config reloading. +# +# A callable that takes a server instance as the sole argument. +# + +def post_fork(server, worker): + server.log.info("Worker spawned (pid: %s)", worker.pid) + +def pre_fork(server, worker): + pass + +def pre_exec(server): + server.log.info("Forked child, re-executing.") + +def when_ready(server): + server.log.info("Server is ready. Spawning workers") + server.log.info(os.getenv('GREET')) +def worker_int(worker): + worker.log.info("worker received INT or QUIT signal") + + ## get traceback info + import threading, sys, traceback + id2name = {th.ident: th.name for th in threading.enumerate()} + code = [] + for threadId, stack in sys._current_frames().items(): + code.append("\n# Thread: %s(%d)" % (id2name.get(threadId,""), + threadId)) + for filename, lineno, name, line in traceback.extract_stack(stack): + code.append('File: "%s", line %d, in %s' % (filename, + lineno, name)) + if line: + code.append(" %s" % (line.strip())) + worker.log.debug("\n".join(code)) + +def worker_abort(worker): + worker.log.info("worker received SIGABRT signal") + +def ssl_context(conf, default_ssl_context_factory): + import ssl + + # The default SSLContext returned by the factory function is initialized + # with the TLS parameters from config, including TLS certificates and other + # parameters. + context = default_ssl_context_factory() + + # The SSLContext can be further customized, for example by enforcing + # minimum TLS version. + context.minimum_version = ssl.TLSVersion.TLSv1_3 + + # Server can also return different server certificate depending which + # hostname the client uses. Requires Python 3.7 or later. + def sni_callback(socket, server_hostname, context): + if server_hostname == "foo.127.0.0.1.nip.io": + new_context = default_ssl_context_factory() + new_context.load_cert_chain(certfile="foo.pem", keyfile="foo-key.pem") + socket.context = new_context + + context.sni_callback = sni_callback + + return context diff --git a/Release/Datastore/DSLogic/DSlogcomp/requirements.txt b/Release/Datastore/DSLogic/DSlogcomp/requirements.txt new file mode 100644 index 0000000..2e1c7bc --- /dev/null +++ b/Release/Datastore/DSLogic/DSlogcomp/requirements.txt @@ -0,0 +1,17 @@ +blinker==1.7.0 +certifi==2024.2.2 +charset-normalizer==3.3.2 +click==8.1.7 +flask==3.0.3 +idna==3.7 +importlib-metadata==7.1.0 +itsdangerous==2.1.2 +Jinja2==3.1.3 +MarkupSafe==2.1.5 +PyYAML==6.0.1 +requests==2.31.0 +urllib3==2.2.1 +werkzeug==3.0.2 +zipp==3.18.1 +gunicorn==22.0.0 + diff --git a/Release/Datastore/DSLogic/kubernetes/depl.yaml b/Release/Datastore/DSLogic/kubernetes/depl.yaml new file mode 100644 index 0000000..fbe6251 --- /dev/null +++ b/Release/Datastore/DSLogic/kubernetes/depl.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: Service +metadata: + name: dslogic + labels: + app: dslogic +spec: + ports: + - name: dsport + port: 5000 + + + + selector: + app: dslogic +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dslog-deploy + labels: + app: dslogic + annotations: + author: "Alessio Romano" + mail: "alessio.romano329@edu.unito.it" + descr: "Flask on gunicorn server app checking datastore state" + other: "undergrad thesis university of Turin" +spec: + replicas: 1 + selector: + matchLabels: + app: dslogic + template: + metadata: + labels: + app: dslogic + spec: + containers: + - name: dslogic + image: romanoa77/dslogic:k8s.0.1.sync + ports: + - containerPort: 5000 + resources: + requests: + cpu: "2" + memory: "300Mi" diff --git a/Release/Datastore/DataBuff/Databuffcomp/.dockerignore b/Release/Datastore/DataBuff/Databuffcomp/.dockerignore new file mode 100644 index 0000000..3f55a74 --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/.dockerignore @@ -0,0 +1,6 @@ +main/adm +main/data +main/log +main/envsetup.sh + + diff --git a/Release/Datastore/DataBuff/Databuffcomp/Dockerfile b/Release/Datastore/DataBuff/Databuffcomp/Dockerfile new file mode 100644 index 0000000..81d8064 --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/Dockerfile @@ -0,0 +1,46 @@ +# syntax=docker/dockerfile:1 +FROM golang:1.22.4-alpine AS build + +WORKDIR /app/class +COPY class . + +WORKDIR /app/main +COPY main . + +#Set the env. variable to use buildkit +#export DOCKER_BUILDKIT=1 + +RUN --mount=type=cache,target=/go/pkg/mod go mod download + +RUN --mount=type=cache,target=/root/.cache/go-build go build -o /app/main/databuff + +FROM alpine:latest + +ENV ADMROOT=appdata/adm/ +ENV ADMFNM=StatDesc.json +ENV DSST=Dsstat.json + +ENV DATAROOT=data/datavar/ +ENV DATANM=data/datavar + +ENV LOGROOT=appdata/log/ +ENV LOGSTREAM=LogStream.json + + +ENV SERVURL=:8080 + +ENV DTVOLROOT=data/ +ENV WCHSZ=10 + + +WORKDIR /app +RUN mkdir appdata +RUN mkdir data +COPY --from=build /app/main/databuff . + +EXPOSE 8080 + + +CMD [ "./databuff" ] + + diff --git a/Release/Datastore/DataBuff/Databuffcomp/class/appmodel/appmodel.go b/Release/Datastore/DataBuff/Databuffcomp/class/appmodel/appmodel.go new file mode 100644 index 0000000..1f80a62 --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/class/appmodel/appmodel.go @@ -0,0 +1,17 @@ +package appmodel + +import ( + "base.url/class/dsstat" + "base.url/class/fbufstat" + "github.com/gin-gonic/gin" +) + +type AbstrApp interface { + GetStat(Table *fbufstat.Bufstat) gin.HandlerFunc + GetLogF() gin.HandlerFunc + PostFile(Bch chan int) gin.HandlerFunc + GetDSdsc(Table *dsstat.DSstat) gin.HandlerFunc + PostDsc(Table *dsstat.DSstat) gin.HandlerFunc + PostClean(Stpt *fbufstat.Bufstat, Table *dsstat.DSstat) gin.HandlerFunc + GetFLst() gin.HandlerFunc +} diff --git a/Release/Datastore/DataBuff/Databuffcomp/class/appstr/appstr.go b/Release/Datastore/DataBuff/Databuffcomp/class/appstr/appstr.go new file mode 100644 index 0000000..eb4ac30 --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/class/appstr/appstr.go @@ -0,0 +1,263 @@ +package appstr + +import ( + "encoding/json" + "net/http" + "os" + "strconv" + "time" + + "base.url/class/dsstat" + "base.url/class/envdef" + "base.url/class/fbufstat" + "base.url/class/fwrite" + "base.url/class/neterr" + "base.url/class/simplelogger" + "github.com/gin-gonic/gin" +) + +type gw struct { + H_data []float64 `json:"h"` + T_data []float64 `json:"t"` +} + +type ConcrH struct{} + +func (h ConcrH) GetStat(Table *fbufstat.Bufstat) gin.HandlerFunc { + + return func(ctx *gin.Context) { + + ctx.JSON(http.StatusOK, Table) + + } + +} + +func (h ConcrH) GetDSdsc(Table *dsstat.DSstat) gin.HandlerFunc { + + return func(ctx *gin.Context) { + + ctx.JSON(http.StatusOK, Table) + + } + +} + +func (h ConcrH) GetLogF() gin.HandlerFunc { + + return func(ctx *gin.Context) { + + var BufSl []simplelogger.LogWrite + var BufLog simplelogger.LogWrite + var bufst []string + + var count int + var resp_code int + var resp any + + bufst, count = fwrite.RFbyLine(envdef.Baselog, envdef.Strmlogn) + + if count != 0 { + + for i := 0; i < count; i++ { + + lineb := []byte(bufst[i]) + json.Unmarshal(lineb, &BufLog) + BufSl = append(BufSl, BufLog) + + } + + resp_code = http.StatusOK + resp = BufSl + } else { + resp_code = http.StatusNoContent + resp = "{}" + } + + ctx.JSON(resp_code, resp) + + } + +} + +func (h ConcrH) PostFile(Bch chan int) gin.HandlerFunc { + + return func(ctx *gin.Context) { + + var Gwbuf gw + + var fgwsize int + var fgwname string + + var err, errb error + var resp_code int + var resp any + + //Stpt.UpdateCnt() + + fgwname = "gwb" + strconv.FormatInt(time.Now().UnixNano(), 10) + ".json" + + // Call BindJSON to bind the received JSON to + // newAlbum. + + errb = ctx.BindJSON(&Gwbuf) + + fgwsize, err = fwrite.AtmWrtJs(envdef.Basedata, fgwname, Gwbuf) + + if (err == nil) && (errb == nil) { + + Bch <- fgwsize + + simplelogger.LogWriteFile(envdef.Baselog, envdef.Strmlogn, 0, fgwsize, fgwname) + + //Updating app status + //Stpt.UpdateCnt() + + resp_code = http.StatusCreated + resp = neterr.New("RES_CREATE", "Written "+string(fgwname)+"for "+strconv.Itoa(fgwsize)+" bytes") + + } else { + resp_code = http.StatusInternalServerError + resp = neterr.New(neterr.CdErrIO, neterr.BodyFIO) + //Stpt.CancCnt() + + } + + ctx.JSON(resp_code, resp) + + } + +} + +func (h ConcrH) PostDsc(Table *dsstat.DSstat) gin.HandlerFunc { + + return func(ctx *gin.Context) { + + var err error + var resp_code int + var resp any + + Table.SetToFrz() + _, err = fwrite.AtmWrtJs(envdef.Baseadm, envdef.DStnm, Table) + + if err == nil { + + resp_code = http.StatusCreated + resp = neterr.New("STAT_UPD", "Table updated") + } else { + resp_code = http.StatusInternalServerError + resp = neterr.New(neterr.CdErrIO, neterr.BodyFIO) + Table.SetToOp() + + } + + ctx.JSON(resp_code, resp) + + } +} + +func (h ConcrH) PostClean(Stpt *fbufstat.Bufstat, Table *dsstat.DSstat) gin.HandlerFunc { + + return func(ctx *gin.Context) { + + var buf fbufstat.Bufstat + var flag bool + var err, derr error + + var resp_code int + var resp any + + flag = false + + now := time.Now() + + if os.Rename(envdef.Basedata, envdef.Basevoldt+now.Format("Mon Jan 2 15:04")) != nil { + + //simplelogger.LogInf(envdef.Baselog, "infolog", "errore nominazione") + flag = true + } + + if os.Mkdir(envdef.Datanm, os.ModePerm) != nil { + flag = true + + //simplelogger.LogInf(envdef.Baselog, "infolog", "errore creazione dir") + + } + + _, err = os.Create(envdef.Baselog + envdef.Strmlogn) + + if err != nil { + + //simplelogger.LogInf(envdef.Baselog, "infolog", "errore log") + + flag = true + } + + Stpt.N_itm = 0 + Stpt.Buff_size = 0 + + buf.N_itm = 0 + buf.Buff_size = 0 + + Table.SetToOp() + Table.User = "databuf" + Table.Token = "databuf" + + _, err = fwrite.AtmWrtJs(envdef.Baseadm, envdef.Baseadmn, buf) + _, derr = fwrite.AtmWrtJs(envdef.Baseadm, envdef.DStnm, Table) + + if (err != nil) || (derr != nil) { + + //simplelogger.LogInf(envdef.Baselog, "infolog", "errore stato") + + flag = true + } + + if flag { + + resp_code = http.StatusInternalServerError + resp = neterr.New(neterr.CdErrIO, neterr.BodyFIO) + + } else { + + resp_code = http.StatusCreated + resp = neterr.New("STAT_UPD", "Buffer cleaned") + + } + + ctx.JSON(resp_code, resp) + + } +} + +func (h ConcrH) GetFLst() gin.HandlerFunc { + + return func(ctx *gin.Context) { + + var Buf fwrite.FList + var resp_code int + var resp any + + dir, err := os.ReadDir(envdef.Basedata) + + if err == nil { + + for _, fs := range dir { + + Buf.NMList = append(Buf.NMList, fs.Name()) + } + + resp_code = http.StatusOK + resp = Buf + + } else { + + resp_code = http.StatusInternalServerError + resp = neterr.New(neterr.CdErrIO, neterr.BodyFIO) + + } + + ctx.JSON(resp_code, resp) + + } +} diff --git a/Release/Datastore/DataBuff/Databuffcomp/class/dsstat/dsstat.go b/Release/Datastore/DataBuff/Databuffcomp/class/dsstat/dsstat.go new file mode 100644 index 0000000..344ee96 --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/class/dsstat/dsstat.go @@ -0,0 +1,81 @@ +package dsstat + +import ( + "encoding/json" + "sync" +) + +type DSstat struct { + Dstatus string `json:"dstatus"` + User string `json:"user"` + + Token string `json:"token"` +} + +type Idt struct { + User string `json:"user"` + + Token string `json:"token"` +} + +var instance *DSstat +var once sync.Once +var mu sync.Mutex + +const opstr = "OPERATIONAL" +const frzstr = "FROZEN" + +func GetInst() *DSstat { + + once.Do(func() { + + instance = &DSstat{opstr, "databuf", "databuf"} + + }) + + return instance +} + +func (Class *DSstat) SetToFrz() { + + mu.Lock() + Class.Dstatus = frzstr + mu.Unlock() + +} + +func (Class *DSstat) SetToOp() { + + mu.Lock() + Class.Dstatus = opstr + mu.Unlock() + +} + +func (Class *DSstat) SetUsr(buf string) { + + mu.Lock() + Class.User = buf + mu.Unlock() + +} + +func (Class *DSstat) SetTok(buf string) { + + mu.Lock() + Class.Token = buf + mu.Unlock() + +} + +func (Class *DSstat) SetStat(buf []byte) { + + var Ftable DSstat + + json.Unmarshal(buf, &Ftable) + + Class.Dstatus = Ftable.Dstatus + Class.Token = Ftable.Token + Class.User = Ftable.User + +} diff --git a/Release/Datastore/DataBuff/Databuffcomp/class/envdef/envdef.go b/Release/Datastore/DataBuff/Databuffcomp/class/envdef/envdef.go new file mode 100644 index 0000000..e20daa4 --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/class/envdef/envdef.go @@ -0,0 +1,45 @@ +package envdef + +import "os" + +// filesystem +const envadm = "ADMROOT" +const envadmn = "ADMFNM" + +const envdata = "DATAROOT" +const envdtnm = "DATANM" +const envlog = "LOGROOT" + +const envlogstrm = "LOGSTREAM" + +const envres = "RESROOT" + +const envdtrt = "DTVOLROOT" + +const envdsst = "DSST" + +// network +const envserv = "SERVURL" +const envch = "WCHSZ" + +//filesystem + +var Baseadm = os.Getenv(envadm) +var Baseadmn = os.Getenv(envadmn) + +var Basedata = os.Getenv(envdata) +var Datanm = os.Getenv(envdtnm) + +var Baselog = os.Getenv(envlog) +var Strmlogn = os.Getenv(envlogstrm) + +var Baseres = os.Getenv(envres) + +var Basevoldt = os.Getenv(envdtrt) + +var DStnm = os.Getenv(envdsst) + +//network + +var Basesrvurl = os.Getenv(envserv) +var Chlen = os.Getenv(envch) diff --git a/Release/Datastore/DataBuff/Databuffcomp/class/fbufstat/fbufstat.go b/Release/Datastore/DataBuff/Databuffcomp/class/fbufstat/fbufstat.go new file mode 100644 index 0000000..b7c7fd8 --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/class/fbufstat/fbufstat.go @@ -0,0 +1,100 @@ +package fbufstat + +import ( + "encoding/json" + "sync" +) + +type Bufstat struct { + N_itm int `json:"n_itm"` + Buff_size int `json:"buff_size"` +} + +var instance *Bufstat +var once sync.Once +var mu sync.Mutex +var nu sync.Mutex + +func GetInst() *Bufstat { + + once.Do(func() { + + instance = &Bufstat{0, 0} + + }) + + return instance +} + +func (Class *Bufstat) SetStat(buf []byte) { + + var Ftable Bufstat + + json.Unmarshal(buf, &Ftable) + + Class.N_itm = Ftable.N_itm + Class.Buff_size = Ftable.Buff_size + +} + +func (Class *Bufstat) GetObj() *Bufstat { + + return Class + +} + +func (Class *Bufstat) UpdateCnt() { + + mu.Lock() + Class.N_itm += 1 + mu.Unlock() + +} + +func (Class *Bufstat) CancCnt() { + + mu.Lock() + Class.N_itm -= 1 + mu.Unlock() + +} + +func (Class Bufstat) GetJSONObj() []byte { + + bt_arr, _ := json.Marshal(Class) + + return bt_arr + +} + +func (Class *Bufstat) UpdateSize(buf int) { + + mu.Lock() + Class.Buff_size += buf + mu.Unlock() + +} + +func (Class *Bufstat) CancSize(buf int) { + + mu.Lock() + Class.Buff_size -= buf + mu.Unlock() + +} + +func (Class *Bufstat) UpdateStMux(bufs int) { + + mu.Lock() + Class.N_itm += 1 + Class.Buff_size += bufs + mu.Unlock() + +} + +func (Class *Bufstat) UpdateSt(bufs int) { + + Class.N_itm += 1 + Class.Buff_size += bufs + +} diff --git a/Release/Datastore/DataBuff/Databuffcomp/class/fwrite/fwrite.go b/Release/Datastore/DataBuff/Databuffcomp/class/fwrite/fwrite.go new file mode 100644 index 0000000..abdb5a7 --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/class/fwrite/fwrite.go @@ -0,0 +1,134 @@ +package fwrite + +/* +The scope of this class is to provide methods for read/write streams +of bytes. Every operation i/o operation should be atomical + +i.e. I receive a stream from endpoints. Write a new file locally +i.e Reading single file +*/ + +import ( + "bufio" + "encoding/json" + "fmt" + "io/fs" + "os" + + "github.com/google/renameio" +) + +type FList struct { + NMList []string `json:"nmlist"` +} + +// If the func literal starts with an uppercase letter +// the function will be public +func StructToByte(buf any) []byte { + + bt_arr, _ := json.Marshal(buf) + + return bt_arr + +} + +func PrintStToF(dest string, filename string, mod fs.FileMode, DtStr any) (int, error) { + + var code error + var fsize int + + buf := StructToByte(DtStr) + fsize = len(buf) + + //0644 read-only mod for other users. + //0666 THE NUMBER OF THE BEAST! Read and write for all! + //Remember WriteFile return nil with success. + code = os.WriteFile(dest+filename, buf, mod) + + return fsize, code + +} + +func PrintStrmToF(dest string, filename string, mod fs.FileMode, DtStr []byte) error { + + //0644 read-only mod for other users. + //0666 Read and write for all + //Remember WriteFile return nil with success. + code := os.WriteFile(dest+filename, DtStr, mod) + + return code + +} + +func FtoStrm(dest string, filename string) ([]byte, error) { + + var code error + var bt_arr []byte + + bt_arr, code = os.ReadFile(dest + filename) + + return bt_arr, code + +} + +func UnFtoStrm(dest string, filename string) ([]byte, error) { + + var bt_arr []byte + var err error + + bt_arr, err = os.ReadFile(dest + filename) + + return bt_arr, err + +} + +func RFbyLine(dest string, filename string) ([]string, int) { + + var buf []string + i := 0 + var code error + + file, code := os.Open(dest + filename) + + if code == nil { + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + line := scanner.Text() + buf = append(buf, line) + i++ + + } + + if scanner.Err() != nil { + + i = 0 + } + + file.Close() + + } + + return buf, i + +} + +func AtmWrtJs(dest string, name string, data any) (int, error) { + + var cbuf []byte + + cbuf, _ = json.Marshal(data) + + t, err := renameio.TempFile("", dest+name) + if err != nil { + return 0, err + } + defer t.Cleanup() + + if count, err := fmt.Fprintf(t, "%s", cbuf); err != nil { + return 0, err + } else { + return count, t.CloseAtomicallyReplace() + } + +} diff --git a/Release/Datastore/DataBuff/Databuffcomp/class/go.mod b/Release/Datastore/DataBuff/Databuffcomp/class/go.mod new file mode 100644 index 0000000..388a133 --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/class/go.mod @@ -0,0 +1,36 @@ +module base.url/class + +go 1.22.2 + +require ( + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gin-gonic/gin v1.10.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/google/renameio v1.0.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/rs/zerolog v1.32.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/Release/Datastore/DataBuff/Databuffcomp/class/go.sum b/Release/Datastore/DataBuff/Databuffcomp/class/go.sum new file mode 100644 index 0000000..ce78be2 --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/class/go.sum @@ -0,0 +1,95 @@ +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v1.0.1 h1:Lh/jXZmvZxb0BBeSY5VKEfidcbcbenKjZFzM/q0fSeU= +github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= +github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/Release/Datastore/DataBuff/Databuffcomp/class/neterr/neterr.go b/Release/Datastore/DataBuff/Databuffcomp/class/neterr/neterr.go new file mode 100644 index 0000000..53e3cf8 --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/class/neterr/neterr.go @@ -0,0 +1,18 @@ +package neterr + +const CdErrIO = "ERRIO" + +const BodyFIO = "Fs error detected" + +type ErrResp struct { + Code string `json:"code"` + Body string `json:"body"` +} + +func New(codstr string, bodystr string) ErrResp { + + Resp := ErrResp{codstr, bodystr} + + return Resp + +} diff --git a/Release/Datastore/DataBuff/Databuffcomp/class/netutil/netutil.go b/Release/Datastore/DataBuff/Databuffcomp/class/netutil/netutil.go new file mode 100644 index 0000000..0c5a8ec --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/class/netutil/netutil.go @@ -0,0 +1,12 @@ +package netutil + +import ( + "base.url/class/fbufstat" + "github.com/gin-gonic/gin" +) + +func SendJSON(Buf fbufstat.Bufstat, Hnd *gin.Context) { + + Hnd.JSON(200, Buf) + +} diff --git a/Release/Datastore/DataBuff/Databuffcomp/class/simplelogger/simplelogger.go b/Release/Datastore/DataBuff/Databuffcomp/class/simplelogger/simplelogger.go new file mode 100644 index 0000000..e8c0497 --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/class/simplelogger/simplelogger.go @@ -0,0 +1,76 @@ +package simplelogger + +import ( + "errors" + "os" + + "github.com/rs/zerolog" +) + +type LogWrite struct { + Level string `json:"level"` + Id int `json:"Id"` + Size int `json:"size"` + Name string `json:"name"` + Time string `json:"time"` + Message string `json:"message"` +} + +func LogWriteFile(dest string, filename string, i int, size int, fname string) error { + + file, err := os.OpenFile( + dest+filename, + os.O_APPEND|os.O_CREATE|os.O_WRONLY, + 0666, + ) + if err != nil { + return err + } + + defer file.Close() + + logger := zerolog.New(file).With().Timestamp().Logger() + + logger.Info().Int("id", i).Int("size", size).Str("name", fname).Msg("Write file") + + return err + +} + +func LogPanic(main_err string, descr string) { + + err := errors.New(main_err) + + logger := zerolog.New(os.Stdout).With().Timestamp().Logger() + + logger.Panic().Err(err).Msg(descr) + +} + +func LogGreet(greeter string) { + + logger := zerolog.New(os.Stdout).With().Timestamp().Logger() + logger.Info().Msg(greeter) + +} + +func LogInf(dest string, filename string, info string) error { + + file, err := os.OpenFile( + dest+filename, + os.O_APPEND|os.O_CREATE|os.O_WRONLY, + 0666, + ) + if err != nil { + return err + } + + defer file.Close() + + logger := zerolog.New(file).With().Timestamp().Logger() + + logger.Info().Msg(info) + + return err + +} diff --git a/Release/Datastore/DataBuff/Databuffcomp/main/adm/Dsstat.json b/Release/Datastore/DataBuff/Databuffcomp/main/adm/Dsstat.json new file mode 100644 index 0000000..fe30c5b --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/main/adm/Dsstat.json @@ -0,0 +1 @@ +{"dstatus":"OPERATIONAL","user":"databuf","token":"databuf"} \ No newline at end of file diff --git a/Release/Datastore/DataBuff/Databuffcomp/main/adm/StatDesc.json b/Release/Datastore/DataBuff/Databuffcomp/main/adm/StatDesc.json new file mode 100644 index 0000000..2c9b2b9 --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/main/adm/StatDesc.json @@ -0,0 +1 @@ +{"n_itm":0,"buff_size":0} diff --git a/Release/Datastore/DataBuff/Databuffcomp/main/envsetup.sh b/Release/Datastore/DataBuff/Databuffcomp/main/envsetup.sh new file mode 100755 index 0000000..4fc6c91 --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/main/envsetup.sh @@ -0,0 +1,18 @@ +#!/bin/bash + + +export ADMROOT=adm/ +export ADMFNM=StatDesc.json + +export DATAROOT=data/ +export DATANM=data + +export LOGROOT=log/ +export LOGSTREAM=LogStream.json + +export RESROOT=static/ + + +export SERVURL=:8080 +export WCHSZ=10 +export DSST=Dsstat.json diff --git a/Release/Datastore/DataBuff/Databuffcomp/main/go.mod b/Release/Datastore/DataBuff/Databuffcomp/main/go.mod new file mode 100644 index 0000000..45e9ff4 --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/main/go.mod @@ -0,0 +1,42 @@ +module base.url/main + +go 1.22.2 + +replace base.url/class => ../class + +require ( + base.url/class v0.0.0-00010101000000-000000000000 + github.com/gin-gonic/gin v1.10.0 +) + +require ( + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/google/renameio v1.0.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/rs/zerolog v1.32.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/Release/Datastore/DataBuff/Databuffcomp/main/go.sum b/Release/Datastore/DataBuff/Databuffcomp/main/go.sum new file mode 100644 index 0000000..b9fadce --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/main/go.sum @@ -0,0 +1,103 @@ +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v1.0.1 h1:Lh/jXZmvZxb0BBeSY5VKEfidcbcbenKjZFzM/q0fSeU= +github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= +github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/Release/Datastore/DataBuff/Databuffcomp/main/log/LogStream.json b/Release/Datastore/DataBuff/Databuffcomp/main/log/LogStream.json new file mode 100644 index 0000000..e69de29 diff --git a/Release/Datastore/DataBuff/Databuffcomp/main/main.go b/Release/Datastore/DataBuff/Databuffcomp/main/main.go new file mode 100644 index 0000000..57f45da --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/main/main.go @@ -0,0 +1,165 @@ +package main + +import ( + "context" + "errors" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + "time" + + "base.url/class/appmodel" + "base.url/class/appstr" + "base.url/class/dsstat" + "base.url/class/envdef" + "base.url/class/fbufstat" + "base.url/class/fwrite" + "base.url/class/simplelogger" + "github.com/gin-gonic/gin" +) + +func main() { + + var nch int + StatDesc := fbufstat.GetInst() + DSDesc := dsstat.GetInst() + + nch, _ = strconv.Atoi(envdef.Chlen) + + wrtch := make(chan int, nch) + msgch := make(chan int) + + if initstat(envdef.Baseadm, envdef.Baseadmn, envdef.DStnm, StatDesc, DSDesc) != nil { + + simplelogger.LogPanic("FATAL ERROR", "FS ERROR") + + } + + simplelogger.LogGreet("DB ready") + + App := appstr.ConcrH{} + + gin.SetMode(gin.ReleaseMode) + + router := gin.Default() + + initapp(router, App, StatDesc, DSDesc, wrtch) + + srv := &http.Server{Addr: envdef.Basesrvurl, Handler: router} + + go worker(wrtch, msgch, StatDesc) + + go func() { + // service connections + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + simplelogger.LogPanic("FATAL ERROR", "NETWORK ERROR") + } + }() + + quitch := make(chan os.Signal, 1) + + signal.Notify(quitch, syscall.SIGINT, syscall.SIGTERM) + //main blocks until a signal is received + <-quitch + + simplelogger.LogGreet("Init shutdown...") + + ct, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := srv.Shutdown(ct); err != nil { + simplelogger.LogPanic("FORCING SHUTDOWN!", "NETWORK ERROR") + } + + // catching ct.Done(). timeout of 5 seconds. + simplelogger.LogGreet("Timeout of 5 seconds...") + <-ct.Done() + + simplelogger.LogGreet("Server exiting.") + msgch <- 0 + simplelogger.LogGreet("Timeout of 10 seconds...") + time.Sleep(10 * time.Second) + simplelogger.LogGreet("Databuff closing...") + +} + +func worker(Bch chan int, Mch chan int, Stpt *fbufstat.Bufstat) { + + var buf int + var err error + + for { + + select { + + case buf = <-Bch: + + Stpt.UpdateSt(buf) + _, err = fwrite.AtmWrtJs(envdef.Baseadm, envdef.Baseadmn, Stpt) + + if err != nil { + + simplelogger.LogPanic("FATAL ERROR", "FS ERROR") + + } + + case <-Mch: + + simplelogger.LogGreet("Main worker thread stopping...") + return + + } + + } + +} + +func initstat(rtdir string, rtfname string, descfname string, Buf *fbufstat.Bufstat, Dsd *dsstat.DSstat) error { + var err, derr, initerr error + var cbuf, dbuf []byte + + initerr = nil + + cbuf, err = fwrite.UnFtoStrm(rtdir, rtfname) + dbuf, derr = fwrite.UnFtoStrm(rtdir, descfname) + + if (err == nil) && (derr == nil) { + + Buf.SetStat(cbuf) + Dsd.SetStat(dbuf) + } else { + + initerr = errors.New("FS ERR") + } + + return initerr +} + +func PostWho(Desc *dsstat.DSstat) gin.HandlerFunc { + + return func(ctx *gin.Context) { + + var buf dsstat.Idt + + ctx.BindJSON(&buf) + + Desc.User = buf.User + Desc.Token = buf.Token + + ctx.Next() + + } +} + +func initapp(SrvPt *gin.Engine, AppRts appmodel.AbstrApp, Buf *fbufstat.Bufstat, Desc *dsstat.DSstat, Chnl chan int) { + + SrvPt.GET("/stat", AppRts.GetStat(Buf)) + SrvPt.GET("/dumpLogF", AppRts.GetLogF()) + SrvPt.GET("/dstat", AppRts.GetDSdsc(Desc)) + SrvPt.POST("/sendF", AppRts.PostFile(Chnl)) + SrvPt.POST("/upddsc", PostWho(Desc), AppRts.PostDsc(Desc)) + SrvPt.POST("/cleanall", AppRts.PostClean(Buf, Desc)) + SrvPt.GET("/dumpF", AppRts.GetFLst()) + +} diff --git a/Release/Datastore/DataBuff/Databuffcomp/main/static/Dsstat.json b/Release/Datastore/DataBuff/Databuffcomp/main/static/Dsstat.json new file mode 100644 index 0000000..fe30c5b --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/main/static/Dsstat.json @@ -0,0 +1 @@ +{"dstatus":"OPERATIONAL","user":"databuf","token":"databuf"} \ No newline at end of file diff --git a/Release/Datastore/DataBuff/Databuffcomp/main/static/LogStream.json b/Release/Datastore/DataBuff/Databuffcomp/main/static/LogStream.json new file mode 100644 index 0000000..e69de29 diff --git a/Release/Datastore/DataBuff/Databuffcomp/main/static/StatDesc.json b/Release/Datastore/DataBuff/Databuffcomp/main/static/StatDesc.json new file mode 100644 index 0000000..2c9b2b9 --- /dev/null +++ b/Release/Datastore/DataBuff/Databuffcomp/main/static/StatDesc.json @@ -0,0 +1 @@ +{"n_itm":0,"buff_size":0} diff --git a/Release/Datastore/DataBuff/Kubernetes/adml.yaml b/Release/Datastore/DataBuff/Kubernetes/adml.yaml new file mode 100644 index 0000000..ae4d747 --- /dev/null +++ b/Release/Datastore/DataBuff/Kubernetes/adml.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: adml +spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: local-path + resources: + requests: + storage: "1000Mi" + + diff --git a/Release/Datastore/DataBuff/Kubernetes/gwdataln.yaml b/Release/Datastore/DataBuff/Kubernetes/gwdataln.yaml new file mode 100644 index 0000000..bbf7773 --- /dev/null +++ b/Release/Datastore/DataBuff/Kubernetes/gwdataln.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: gwdataln +spec: + accessModes: [ "ReadWriteMany" ] + storageClassName: longhorn + resources: + requests: + storage: "1000Mi" + + + + diff --git a/Release/Datastore/DataBuff/Kubernetes/state.yaml b/Release/Datastore/DataBuff/Kubernetes/state.yaml new file mode 100644 index 0000000..b10428f --- /dev/null +++ b/Release/Datastore/DataBuff/Kubernetes/state.yaml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: Service +metadata: + name: databuff + labels: + app: databuff +spec: + ports: + - name: ginport + port: 8080 + + clusterIP: None + + selector: + app: databuff +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: databuff + annotations: + author: "Alessio Romano" + mail: "alessio.romano329@edu.unito.it" + descr: "datastore storing GW data written in go" + other: "undergrad thesis university of Turin" +spec: + serviceName: databuff + replicas: 1 + selector: + matchLabels: + app: databuff + template: + metadata: + labels: + app: databuff + spec: + containers: + - name: databuff + image: romanoa77/databuff:alpha.k8s.1.2 + #command: ["sleep","infinity"] + ports: + - containerPort: 8080 + name: ginport + resources: + requests: + cpu: "2" + memory: "300Mi" + limits: + memory: "300Mi" + volumeMounts: + - name: adata + mountPath: /app/appdata + - name: gdata + mountPath: /app/data + + volumes: + - name: adata + persistentVolumeClaim: + claimName: adml + - name: gdata + persistentVolumeClaim: + claimName: gwdataln + + + + diff --git a/Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/Dockerfile b/Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/Dockerfile new file mode 100644 index 0000000..301b34b --- /dev/null +++ b/Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/Dockerfile @@ -0,0 +1,26 @@ +# syntax=docker/dockerfile:1 +FROM python:3.8.10-slim-buster + +ENV DS_STAT=bufstat +ENV DS_SEND=stream +ENV DS_FREEZE=freezeds +ENV DS_BURL=http://dslogic.glitchflow.svc.cluster.local:5000/ +ENV DS_FLUSH=flushbuf +ENV DS_DESC=dsdesc +ENV DS_DUMPF=dumpF +ENV MAX_SIZE=2000 + + +WORKDIR /code + + +COPY ./requirements.txt /code/requirements.txt + + +RUN --mount=type=cache,target=/root/.cache/pip pip install -r /code/requirements.txt + + +COPY ./app /code/app + + +CMD ["fastapi", "run", "app/main.py", "--port", "8000"] \ No newline at end of file diff --git a/Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/app/Appfconf.py b/Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/app/Appfconf.py new file mode 100644 index 0000000..03ae4bc --- /dev/null +++ b/Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/app/Appfconf.py @@ -0,0 +1,25 @@ +import os + +""" + +ENV. TO BE DEFINED INSIDE THE CONTAINER IMG + +ENDP_STAT +ENDP_DUMP +ENDP_SEND +DB_BURL +""" + +env_stat=os.environ['DS_STAT'] + +env_send=os.environ['DS_SEND'] +env_freeze=os.environ['DS_FREEZE'] +env_burl=os.environ['DS_BURL'] +env_clean=os.environ['DS_FLUSH'] +env_desc=os.environ['DS_DESC'] +env_dumpbuf=os.environ['DS_DUMPF'] +env_max_size=os.environ['MAX_SIZE'] + + + + diff --git a/Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/app/__init__.py b/Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/app/main.py b/Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/app/main.py new file mode 100644 index 0000000..b3e1ec5 --- /dev/null +++ b/Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/app/main.py @@ -0,0 +1,129 @@ +import httpx +from . import Appfconf +from fastapi import FastAPI, Request,status +from fastapi.responses import RedirectResponse +from contextlib import asynccontextmanager +from pydantic import BaseModel +from typing import List +import sys +import json + + +class FrzMsg(BaseModel): + user: str + token:str + + +class GWData(BaseModel): + h: List[float] + t: List[float] + +class EndMsg(BaseModel): + msg: str + + + +@asynccontextmanager +async def lifespan(app: FastAPI): + app.requests_client = httpx.AsyncClient() + yield + await app.requests_client.aclose() + +app = FastAPI(lifespan=lifespan) + +@app.get("/stats") +async def get_ds_stats(request: Request): + requests_client = request.app.requests_client + response = await requests_client.get(Appfconf.env_burl+Appfconf.env_stat) + return response.json() + +@app.get("/dumpbuf") +async def get_ds_buffs(request: Request): + requests_client = request.app.requests_client + response = await requests_client.get(Appfconf.env_burl+Appfconf.env_dumpbuf) + return response.json() + + + + +@app.get("/desc") +async def get_ds_desc(request: Request): + requests_client = request.app.requests_client + response = await requests_client.get(Appfconf.env_burl+Appfconf.env_desc) + buf= response.json() + + desc=buf['dstatus'] + + output={"resp":desc} + + return output + +@app.get("/dspage") +async def redirect_ds_statpg(): + return RedirectResponse(Appfconf.env_burl,status_code=status.HTTP_303_SEE_OTHER) + +@app.post("/train") +async def trainsign(request: Request,msg:FrzMsg): + requests_client = request.app.requests_client + buf={'user':msg.user,'token':msg.token} + buf=json.dumps(buf) + response = await requests_client.post(Appfconf.env_burl+Appfconf.env_freeze, + data=buf) + + + + + if(response.status_code==httpx.codes.CREATED): + buf=response.json() + n_file=buf['n_itm'] + n_byte=buf['buff_size'] + + + output={"resp":'FROZEN',"n_f":n_file,"bt_wrt":n_byte} + + + + + else: + + output={"resp":'WAITING'} + + + + return output + + +@app.post("/streamdata") +async def streamdat(request: Request,stream:GWData): + requests_client = request.app.requests_client + buf={'h':stream.h,'t':stream.t} + buf=json.dumps(buf) + response = await requests_client.post(Appfconf.env_burl+Appfconf.env_send,data=buf) + + if(response.status_code==httpx.codes.CREATED): + output={"resp":'CREATED',"body":response.text} + else: + + output={"resp":'FROZEN',"body":response.text} + + + + return output + + +@app.post("/cleanbuf") +async def clean(request: Request,inp:EndMsg): + requests_client = request.app.requests_client + buf={'msg':inp.msg} + buf=json.dumps(buf) + response = await requests_client.post(Appfconf.env_burl+Appfconf.env_clean,data=buf) + + if(response.status_code==httpx.codes.CREATED): + output={"resp":'CLEANED',"body":response.text} + else: + + output={"resp":'FS ERR',"body":response.text} + + + + return output diff --git a/Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/envsetup.sh b/Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/envsetup.sh new file mode 100755 index 0000000..4edcb80 --- /dev/null +++ b/Release/Datastore/GlitchFlowAPI/GlitchFlowAPIcomp/envsetup.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +###va lanciato con source + +<
+ + +> ADMROOT + +Directory containing metadata.The path is expected to be mountpoint/directory, mountpoint is where the kubernetes volume is mounted. + +>ADMFNM + + Name of the metadata file relating to the gw stored data. + +>DSST + +Name of the metadata file relating to the datastore state. + +>DATAROOT + +Directory containing gw data. The path is expected to be mountpoint/directory. + +>DATANM + +Directory containing gw data(duplicate). + +>LOGROOT + +Log directory location. + +>LOGSTREAM + +Log file name. + +>SERVURL + +Port where the service listens for connections. + +>DTVOLROOT + +Directory containing gw data inside the volume. + +>WCHSZ + +Requests buffer size. + +### 4. Kubernetes Setup + +Several object configuration files are used:

+ +-state.yaml + +Defines a statefulset primitive for the service. + +-adml.yaml + +Defines a volume storing application metadata and the log file. + +-gwdataln.yaml + +Defines the volume storing gw data. + +### 5. Commands +To compile the source first install the required go libraries, from the main directory + +> go mod download + +then give the build command + +> go build -o databuff + +### 6. Directories description + +Go is similar to the C programming language so a main function is required. The main directory contains the main function definition with the addition of copies of the application data. The class directory contains the classes definitions. Go does not provide a class data type like in C++. A program can follow the OOP paradigm using +some language features like the struct data type and modules. + +### 7. Application data + +Inside a container the application will have the following directory tree: + + . + ├── application/ + │ └── databuff + ├── datavar/ + │ └── data/ + └── appdata/ + ├── adm/ + │ ├── StatDesc.json + │ └── Dsstat.json + └── log/ + └── LogStream.json + +- databuff is the application executable +- The datavar and appdata directories are mountpoints for their respective kubernetes volume +- The data directory contains stored gw data +- The adm directory contains metadata relating to the system +- The log directory contains the log file + +Regarding the metadata: + +-StatDesc.json contains the following fields: + + - n_itm: the number of files stored + - buff_size: total number of bytes written + +-Dsstat.json contains the following fields: + + - dstatus: datastore state. It can assume the OPERATIONAL/FROZEN values + - user: client identification string + - token: identity token + +The last fields are related to a rudimentary client identification mechanism. + +For a local installation the directory tree is the same as the one inside the Databuffcomp directory. + +### 8. Endpoints Description + +GET methods + +> /stat + +The service will send the content of the StatDesc.json file in the JSON format. + +> /dumpLogF + + +The service will send the content of the log file in the JSON format. + +> /dstat + +The service will send the content of the Dsstat.json file. + +> /dumpF + +The service will send a list of the stored files using the JSON format. + + + +POST methods + +> /sendF + +This method accepts a JSON file containing gw data. The received data will be stored on disk. + +> /upddsc + +The service will update the Dsstat.json file with the content of the received JSON file. This is the the method used by the Datastore logic to freeze the Datastore. + +> /cleanall + +This method will unfreeze the datastore. All written data will be stored inside a directory named as "DAY HOUR MIN", the date refers to when the request has been accepted. + +## Datastore Logic + +The Datastore Logic is a web application written with Flask. It performs the datastore operations using the functionalities of the Data Buffer service. The service runs inside a gunicorn server. + +### 1. Requirements + +The following python packages are required:

+ + blinker==1.7.0 + certifi==2024.2.2 + charset-normalizer==3.3.2 + click==8.1.7 + flask==3.0.3 + idna==3.7 + importlib-metadata==7.1.0 + itsdangerous==2.1.2 + Jinja2==3.1.3 + MarkupSafe==2.1.5 + PyYAML==6.0.1 + requests==2.31.0 + urllib3==2.2.1 + werkzeug==3.0.2 + zipp==3.18.1 + gunicorn==22.0.0 + +### 2. Configuration + +Environment variables are defined inside the Dockerfile. For a local installation the envsetup.sh file can be used. In addition the gunicorn server can be configured +using the gunicorn.conf.py file. See gunicorn documentation https://docs.gunicorn.org/en/22.0.0/settings.html . +### 3. Container Setup +Environment variables description

+ +> ENDP_STAT + +Name of the Data Buffer endpoint for getting the content of the StatDesc.json file. + +> ENDP_DUMP + +Name of the Data Buffer endpoint for getting the content of the log file. + +> ENDP_DUMPF + +Name of the Data Buffer endpoint for getting the list of the stored gw data files. + +> ENDP_SEND + +Name of the Data Buffer endpoint used for sending gw data. + +> ENDP_DESC + +Name of the Data Buffer endpoint for getting the content of the Dsstat.json file. + +> ENDP_STAT + +Name of the Data Buffer endpoint for updating the content of the StatDesc.json file. + +> ENDP_CLEAN + +Name of the Data Buffer endpoint for unfreezing the datastore. + +> DB_BURL + +Data Buffer url. + +> MAX_SIZE + +Size of the stored gw data required for freezing the datastore. +### 4. Kubernetes Setup + +-depl.yaml + +Defines a deployment for the service. +### 5. Commands + +If you want to launch the service from a local installation use the command + +> gunicorn -c /path to gunicorn.conf.py app:create_app('default') + +for example: + +> gunicorn -c /etc/gunicorn.conf.py app:create_app('default') + +### 6. Directory Description + + . + ├── app/ + ├── main/ + └── static/ + └── templates/ + +- main contains methods definition for serving requests +- static contains resources used in html pages +- templates contains html pages + +### 7. Endpoint Description + +POST methods + +> /flushbuf + +Method for cleaning datastore + +> /freezeds + +Method for freezing datastore + +> /stream + +Method for sending gw data + + +GET methods + +> /dsdesc + +Method for getting the content of the Dsstat.json + +> /dumpflog + + +Method for getting the content of the log file + +> /dumpf + +Method for getting the list of the stored gw data files + +> / + +Datastore status page + +> /bufstat + +Method for getting the content of the StatDesc.json + +## GlitchflowAPI +API layer of the infrastructure. Developed using the FastAPI Python framework. + +### 1. Requirements +The following python packages are required: + + fastapi==0.111.0 + httpx==0.27.0 + pydantic==2.7.4 + +### 2. Configuration +Environment variables are defined inside the Dockerfile. For a local installation the envsetup.sh file can be used. +FastAPI runs upon an Uvicorn webserver, see the official documentation for more details https://fastapi.tiangolo.com/deployment/server-workers/#multiple-workers + +### 3. Container Setup + +> DS_STAT + +Name of the Datastore Logic endpoint for getting the content of the StatDesc.json file. + + +> DS_DUMPF + +Name of the Datastore Logic for getting the list of the stored gw data files. + +> DS_SEND + +Name of the Datastore Logic endpoint used for sending gw data. + +> DS_DESC + +Name of the Datastore Logic endpoint for getting the content of the Dsstat.json file. + +> DS_FREEZE + +Name of the Datastore Logic endpoint for freezzing the datastore. + +> DS_FLUSH + +Name of the Data Buffer endpoint for unfreezing the datastore. + +> DS_BURL + +Datastore Logic url. + +> MAX_SIZE + +Size of the stored gw data required for freezing the datastore. +### 4. Kubernetes Setup +-deplapi.yaml + +Defines a deployment for the service. + +### 5. Commands +To launch the service on a local installation use the command + +> fastapi run app/main.py --port 8000 + +### 6. API Description + +GET methods + +>/stats + +Retrieve the content of the StatDesc.json file. + + {"n_itm": NUMBER OF FILES STORED, "buff_size": TOTAL SIZE OF STORED DATA} + +>/dumpbuf + +Retrieve a list of the stored gwdata files. + + {"nmlist":[ARRAY OF STRINGS]} + +>/desc + +Datastore status. + + {"resp": OPERATIONAL/FROZEN} + +>/dspage + +Datastore status page. + +POST methods + +> /train + +Datastore freezing signal. The following JSON file is expected to be sent with the request + + {"user": CLIENT ID,"token": IDENTITY TOKEN} + +in case the required batch of data was collected the response will be: + + {"resp":'FROZEN',"n_f": NUMBER OF FILES STORED,"bt_wrt": TOTAL BYTE WRITTEN} + +instead if the datastore is still waiting for data + + {"resp":'WAITING'} + +> /streamdata + +API for sending data. The expected timeseries is: + + {"h":STRAIN ARRAY,"t":[t0,dt]} + +for a stored file the response is: + + {"resp":'CREATED',"body": ADDITIONAL INFORMATION} + +if the datastore is frozen the response will be: + + {"resp":'FROZEN',"body": ADDITIONAL INFORMATION} + +> /cleanbuf + +API for unfreezing the datastore. The following JSON file is expected to be sent with the request + + {"msg": THE CONTENT IS NOT RELEVANT AT THE MOMENT} + +## GWclient + +This is a simple python script for sending data from gw public catalogues to the datastore. During platform tests it has been executed from a shell interacting with a pod. +It relies on the gwpy framework. + +### Requirements + + aiohttp==3.10.3 + gwosc==0.7.1 + gwpy==3.0.8 + numpy==1.24.4 + +### Container setup + +For tests the image of the pod has been my custom docker image gwpyimg:0.1.rc . + +### Usage + +Copy the content of the Gw client directory inside a pod. Configure the script using the envsetup.sh file then execute the python script. + +### Configuration + +> GWID + +Gravitational waves identifier. + +> DTCID + +Interferometer identifier (see gwpy documentation). + +> INF +> SUP + +Data will be sent as an interval (INF,SUP) around the gps of the gravitational waves. + +> SRV + +Url of the API transmitting data. + +## Client + +A collection of shell scripts for sending requests to the datastore. They can be used like the gwclient script. + +## Airflow DAG + +The following API are used: + +>train + +>stats + +A docker image containing the preprocessing software is required. + + + + + + diff --git a/Release/gwclient/dockimg/gwpyimg/0.1.rc/Dockerfile b/Release/gwclient/dockimg/gwpyimg/0.1.rc/Dockerfile new file mode 100644 index 0000000..c6226cf --- /dev/null +++ b/Release/gwclient/dockimg/gwpyimg/0.1.rc/Dockerfile @@ -0,0 +1,39 @@ +# syntax=docker/dockerfile:1 +FROM romanoa77/gwpyimg:0.1 AS build + +ENV PATH=/opt/venv/bin:$PATH +ENV PATH="/root/.cargo/bin:$PATH" + + + + +RUN apt-get install -y libhdf5-dev +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +RUN rustup update + + + + + + +COPY requirements5.txt . + + + +RUN --mount=type=cache,target=/root/.cache/pip pip install \ +--no-cache-dir \ +--compile \ +--global-option="-Os" \ +--global-option="-gO" \ +--global-option="-Wl,--strip-all" \ +-r requirements5.txt + +FROM python:3.10-slim + +ENV PATH=/opt/venv/bin:$PATH +COPY --from=build /opt/venv /opt/venv +COPY --from=build /usr/lib/x86_64-linux-gnu/ /usr/lib/x86_64-linux-gnu/ + + + +CMD [ "sleep", "infinity" ] diff --git a/Release/gwclient/dockimg/gwpyimg/0.1.test/Dockerfile b/Release/gwclient/dockimg/gwpyimg/0.1.test/Dockerfile new file mode 100644 index 0000000..664323a --- /dev/null +++ b/Release/gwclient/dockimg/gwpyimg/0.1.test/Dockerfile @@ -0,0 +1,22 @@ +# syntax=docker/dockerfile:1 +FROM romanoa77/scibase:0.2 AS builder + +ENV PATH=/opt/venv/bin:$PATH + + + +COPY requirements3.txt . + + + +RUN --mount=type=cache,target=/root/.cache/pip pip install \ +--no-cache-dir \ +--compile \ +--global-option="-Os" \ +--global-option="-gO" \ +--global-option="-Wl,--strip-all" \ +-r requirements3.txt + + + +CMD [ "sleep", "infinity" ] diff --git a/Release/gwclient/dockimg/gwpyimg/0.1/Dockerfile b/Release/gwclient/dockimg/gwpyimg/0.1/Dockerfile new file mode 100644 index 0000000..05fd90d --- /dev/null +++ b/Release/gwclient/dockimg/gwpyimg/0.1/Dockerfile @@ -0,0 +1,22 @@ +# syntax=docker/dockerfile:1 +FROM romanoa77/scibase:0.3 AS builder + +ENV PATH=/opt/venv/bin:$PATH + + + +COPY requirements4.txt . + + + +RUN --mount=type=cache,target=/root/.cache/pip pip install \ +--no-cache-dir \ +--compile \ +--global-option="-Os" \ +--global-option="-gO" \ +--global-option="-Wl,--strip-all" \ +-r requirements4.txt + + + +CMD [ "sleep", "infinity" ] diff --git a/Release/gwclient/dockimg/pytools/Dockerfile b/Release/gwclient/dockimg/pytools/Dockerfile new file mode 100644 index 0000000..f6f9422 --- /dev/null +++ b/Release/gwclient/dockimg/pytools/Dockerfile @@ -0,0 +1,19 @@ +# syntax=docker/dockerfile:1 +FROM python:3.10 AS comp + +RUN mkdir build + +WORKDIR /build + +RUN apt-get -y update && apt-get install -y gfortran + +RUN wget http://www.cmake.org/files/v3.15/cmake-3.15.0.tar.gz && \ +tar xzf cmake-3.15.0.tar.gz + +WORKDIR /build/cmake-3.15.0 +RUN ./bootstrap + +RUN make && make install + + +CMD [ "sleep", "infinity" ] diff --git a/Release/gwclient/dockimg/scibase/0.1/Dockerfile b/Release/gwclient/dockimg/scibase/0.1/Dockerfile new file mode 100644 index 0000000..fb280c5 --- /dev/null +++ b/Release/gwclient/dockimg/scibase/0.1/Dockerfile @@ -0,0 +1,22 @@ +# syntax=docker/dockerfile:1 +FROM romanoa77/pytools:0.1 AS compp + + + +RUN python3 -m venv /opt/venv + +ENV PATH=/opt/venv/bin:$PATH +COPY requirements.txt . + +RUN pip install --no-cache-dir --upgrade pip + + +RUN --mount=type=cache,target=/root/.cache/pip pip install \ +--no-cache-dir \ +--compile \ +--global-option="-Os" \ +--global-option="-gO" \ +--global-option="-Wl,--strip-all" \ +-r requirements.txt + +CMD [ "sleep", "infinity" ] diff --git a/Release/gwclient/dockimg/scibase/0.2/Dockerfile b/Release/gwclient/dockimg/scibase/0.2/Dockerfile new file mode 100644 index 0000000..70243ee --- /dev/null +++ b/Release/gwclient/dockimg/scibase/0.2/Dockerfile @@ -0,0 +1,40 @@ +# syntax=docker/dockerfile:1 +FROM romanoa77/scibase:0.1 AS compp1 + +ENV PATH=/opt/venv/bin:$PATH + +RUN apt-get install -y libopenblas-dev liblapack-dev pkg-config python3-dev + +COPY requirements2.txt . + + + +RUN --mount=type=cache,target=/root/.cache/pip pip install \ +--no-cache-dir \ +--compile \ +--global-option="-Os" \ +--global-option="-gO" \ +--global-option="-Wl,--strip-all" \ +-r requirements2.txt + + + +COPY requirements1.txt . + + + +RUN --mount=type=cache,target=/root/.cache/pip pip install \ +--no-cache-dir \ +--compile \ +--global-option="-Os" \ +--global-option="-gO" \ +--global-option="-Wl,--strip-all" \ +-r requirements1.txt + + + + + + + +CMD [ "sleep", "infinity" ] diff --git a/Release/gwclient/dockimg/scibase/0.3/Dockerfile b/Release/gwclient/dockimg/scibase/0.3/Dockerfile new file mode 100644 index 0000000..90c995b --- /dev/null +++ b/Release/gwclient/dockimg/scibase/0.3/Dockerfile @@ -0,0 +1,22 @@ +# syntax=docker/dockerfile:1 +FROM romanoa77/scibase:0.2 AS compp2 + +ENV PATH=/opt/venv/bin:$PATH + + + +COPY requirements3.txt . + + + +RUN --mount=type=cache,target=/root/.cache/pip pip install \ +--no-cache-dir \ +--compile \ +--global-option="-Os" \ +--global-option="-gO" \ +--global-option="-Wl,--strip-all" \ +-r requirements3.txt + + + +CMD [ "sleep", "infinity" ] diff --git a/Release/gwclient/envsetup.sh b/Release/gwclient/envsetup.sh new file mode 100755 index 0000000..fc76736 --- /dev/null +++ b/Release/gwclient/envsetup.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +###va lanciato con source + + + + + +export GWID=GW150914 +export DTCID=L1 +export INF=13 +export SUP=16 + +export SRV=http://localhost:8000/streamdata + + + + diff --git a/Release/gwclient/gwclient.yaml b/Release/gwclient/gwclient.yaml new file mode 100644 index 0000000..1b07e06 --- /dev/null +++ b/Release/gwclient/gwclient.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gwclient +spec: + containers: + - name: gwclient + image: romanoa77/gwpyimg:0.1.rc + + diff --git a/Release/gwclient/lastrd.txt b/Release/gwclient/lastrd.txt new file mode 100644 index 0000000..573541a --- /dev/null +++ b/Release/gwclient/lastrd.txt @@ -0,0 +1 @@ +0 diff --git a/Release/gwclient/main.py b/Release/gwclient/main.py new file mode 100644 index 0000000..9c6a551 --- /dev/null +++ b/Release/gwclient/main.py @@ -0,0 +1,114 @@ +from gwosc.datasets import event_gps +from gwpy.timeseries import TimeSeries +from astropy.units.quantity import Quantity as q +import numpy as np +import aiohttp +import asyncio +import os + +env_gwid=os.environ['GWID'] +env_dtcid=os.environ['DTCID'] +env_inf=float(os.environ['INF']) +env_sup=float(os.environ['SUP']) +env_srv=os.environ['SRV'] + +N_BUFFS=3 +BUFF_SZ=300 + + + +def createBuff(pointer,arr_data,buffer): + + for i in range(BUFF_SZ): + indx=pointer+i + buffer.append(arr_data[indx]) + + return indx + + +async def sendGW(url,gwdata): + + + async with aiohttp.ClientSession() as session: + async with session.post(url,json=gwdata) as response: + + output = await response.text() + print(output) + + + +async def parallPost(GWS): + + tasks = [asyncio.create_task(sendGW(env_srv,GW)) for GW in GWS] + await asyncio.gather(*tasks) + + + + + + +def main(): + + + f=open("lastrd.txt","r") + last_pt=f.readline() + f.close() + datapt=int(last_pt) + gps=event_gps(env_gwid) + data=TimeSeries.fetch_open_data(env_dtcid,gps+env_inf,gps+env_sup) + + + + + sdt=data.dt.value + st0=0.0+sdt*(datapt+1) + + print("Size for", env_gwid,"data",": ",len(data.value)*8," Byte" ) + print ("Array length is: ",len(data.value)) + + + buff1=[] + buff2=[] + buff3=[] + + datapt=createBuff(datapt,data.value,buff1)+1 + datapt=createBuff(datapt,data.value,buff2)+1 + datapt=createBuff(datapt,data.value,buff3)+1 + + gwstruct1={"h":buff1,"t":[st0,sdt]} + st0=st0+sdt*BUFF_SZ + + + + gwstruct2={"h":buff2,"t":[st0,sdt]} + st0=st0+sdt*BUFF_SZ + + gwstruct3={"h":buff2,"t":[st0,sdt]} + + gwlist=[gwstruct1,gwstruct2,gwstruct3] + + asyncio.run(parallPost(gwlist)) + + f=open("lastrd.txt","w") + f.write(str(datapt)) + f.close() + + + + + + + + + + + + + +if __name__=="__main__": + main() + + + + + diff --git a/Release/gwclient/requirements.txt b/Release/gwclient/requirements.txt new file mode 100644 index 0000000..b56e875 --- /dev/null +++ b/Release/gwclient/requirements.txt @@ -0,0 +1,4 @@ +aiohttp==3.10.3 +gwosc==0.7.1 +gwpy==3.0.8 +numpy==1.24.4