Develop

.github/workflows/go.yaml

@@ -0,0 +1,71 @@
+name: Go
+
+on:
+  push:
+    branches: [ main ]
+    paths-ignore:
+      - '**/README.md'
+      - '.github/**'
+  pull_request:
+    branches: [ main ]
+    paths-ignore:
+      - '**/README.md'
+      - '.github/**'
+
+env:
+  REGISTRY: ghcr.io/terratensor
+  REGISTRY_HOST: ghcr.io
+  ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Docker Login
+        uses: azure/docker-login@v1
+        with:
+          login-server: ${{ env.REGISTRY_HOST }}
+          username: audetv
+          password: ${{ github.token }}
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.22.2
+
+      - name: Build
+        run: go build -v ./...
+
+      - name: Test
+        run: go test -v ./...
+
+      - name: Inject branch slug
+        uses: rlespinasse/github-slug-action@v3.x
+
+      - name: Set image tag
+        run: echo "::set-env name=IMAGE_TAG::${{ env.GITHUB_REF_SLUG }}-${GITHUB_RUN_NUMBER}"
+
+      - name: Build
+        run: make docker-build
+
+      - name: Push
+        run: make push
+        if: ${{ github.ref == 'refs/heads/main' }}
+
+      - name: Install SSH key
+        uses: shimataro/ssh-key-action@v2
+        with:
+          key: ${{ secrets.PRODUCTION_KEY }}
+          known_hosts: ${{ secrets.PRODUCTION_HOST }}
+        if: ${{ github.ref == 'refs/heads/main' }}
+
+      - name: Deploy
+        run: make deploy
+        env:
+          BUILD_NUMBER: ${GITHUB_RUN_NUMBER}
+          HOST: ${{ secrets.PRODUCTION_HOST }}
+          PORT: ${{ secrets.PRODUCTION_PORT }}
+        if: ${{ github.ref == 'refs/heads/main' }}

.gitignore

@@ -0,0 +1,2 @@
+.idea
+/config/local.yaml

Makefile

@@ -0,0 +1,56 @@
+init: init-ci
+init-ci:
+	docker-pull docker-build docker-up
+up: docker-up
+down: docker-down
+restart: down up
+
+test:
+	go test -v ./...
+
+lint:
+	golangci-lint run -v
+
+docker-up:
+	docker compose up -d
+docker-down:
+	docker compose down --remove-orphans
+
+docker-down-clear:
+	docker compose down -v --remove-orphans
+
+docker-pull:
+	docker compose pull
+
+docker-build:
+	docker compose build --pull
+
+dev-docker-build:
+	REGISTRY=localhost IMAGE_TAG=main-1 make docker-build
+
+docker-build: docker-build-service docker-build-parser
+
+docker-build-service:
+	DOCKER_BUILDKIT=1 docker --log-level=debug build --pull --build-arg BUILDKIT_INLINE_CACHE=1 \
+    	--tag ${REGISTRY}/svodd-server-service:${IMAGE_TAG} \
+    	--file ./docker/service/Dockerfile .
+
+docker-build-parser:
+	DOCKER_BUILDKIT=1 docker --log-level=debug build --pull --build-arg BUILDKIT_INLINE_CACHE=1 \
+    	--tag ${REGISTRY}/svodd-server-parser:${IMAGE_TAG} \
+    	--file ./docker/parser/Dockerfile .
+
+push:
+	docker push ${REGISTRY}/svodd-server-service:${IMAGE_TAG}
+	docker push ${REGISTRY}/svodd-server-parser:${IMAGE_TAG}
+
+deploy:
+	ssh -o StrictHostKeyChecking=no deploy@${HOST} -p ${PORT} 'docker network create --driver=overlay traefik-public || true'
+	ssh -o StrictHostKeyChecking=no deploy@${HOST} -p ${PORT} 'docker network create --driver=overlay svodd-network || true'
+	ssh -o StrictHostKeyChecking=no deploy@${HOST} -p ${PORT} 'rm -rf svodd-server-service_${BUILD_NUMBER} && mkdir svodd-server-service_${BUILD_NUMBER}'
+
+	envsubst < docker-compose-production.yml > docker-compose-production-env.yml
+	scp -o StrictHostKeyChecking=no -P ${PORT} docker-compose-production-env.yml deploy@${HOST}:svodd-server-service_${BUILD_NUMBER}/docker-compose.yml
+	rm -f docker-compose-production-env.yml
+
+	ssh -o StrictHostKeyChecking=no deploy@${HOST} -p ${PORT} 'cd svodd-server-service_${BUILD_NUMBER} && docker stack deploy --compose-file docker-compose.yml svodd-server-service --with-registry-auth --prune'

README.md

@@ -1 +1,6 @@
-# svodd-server
+# svodd-server
+
+Перед запуском необходимо создать network
+```
+docker network create --driver=gridge svodd-server-net || true
+```

certs/russian_trusted_root_ca_pem.crt

@@ -0,0 +1,33 @@
+-----BEGIN CERTIFICATE-----
+MIIFwjCCA6qgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwcDELMAkGA1UEBhMCUlUx
+PzA9BgNVBAoMNlRoZSBNaW5pc3RyeSBvZiBEaWdpdGFsIERldmVsb3BtZW50IGFu
+ZCBDb21tdW5pY2F0aW9uczEgMB4GA1UEAwwXUnVzc2lhbiBUcnVzdGVkIFJvb3Qg
+Q0EwHhcNMjIwMzAxMjEwNDE1WhcNMzIwMjI3MjEwNDE1WjBwMQswCQYDVQQGEwJS
+VTE/MD0GA1UECgw2VGhlIE1pbmlzdHJ5IG9mIERpZ2l0YWwgRGV2ZWxvcG1lbnQg
+YW5kIENvbW11bmljYXRpb25zMSAwHgYDVQQDDBdSdXNzaWFuIFRydXN0ZWQgUm9v
+dCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMfFOZ8pUAL3+r2n
+qqE0Zp52selXsKGFYoG0GM5bwz1bSFtCt+AZQMhkWQheI3poZAToYJu69pHLKS6Q
+XBiwBC1cvzYmUYKMYZC7jE5YhEU2bSL0mX7NaMxMDmH2/NwuOVRj8OImVa5s1F4U
+zn4Kv3PFlDBjjSjXKVY9kmjUBsXQrIHeaqmUIsPIlNWUnimXS0I0abExqkbdrXbX
+YwCOXhOO2pDUx3ckmJlCMUGacUTnylyQW2VsJIyIGA8V0xzdaeUXg0VZ6ZmNUr5Y
+Ber/EAOLPb8NYpsAhJe2mXjMB/J9HNsoFMBFJ0lLOT/+dQvjbdRZoOT8eqJpWnVD
+U+QL/qEZnz57N88OWM3rabJkRNdU/Z7x5SFIM9FrqtN8xewsiBWBI0K6XFuOBOTD
+4V08o4TzJ8+Ccq5XlCUW2L48pZNCYuBDfBh7FxkB7qDgGDiaftEkZZfApRg2E+M9
+G8wkNKTPLDc4wH0FDTijhgxR3Y4PiS1HL2Zhw7bD3CbslmEGgfnnZojNkJtcLeBH
+BLa52/dSwNU4WWLubaYSiAmA9IUMX1/RpfpxOxd4Ykmhz97oFbUaDJFipIggx5sX
+ePAlkTdWnv+RWBxlJwMQ25oEHmRguNYf4Zr/Rxr9cS93Y+mdXIZaBEE0KS2iLRqa
+OiWBki9IMQU4phqPOBAaG7A+eP8PAgMBAAGjZjBkMB0GA1UdDgQWBBTh0YHlzlpf
+BKrS6badZrHF+qwshzAfBgNVHSMEGDAWgBTh0YHlzlpfBKrS6badZrHF+qwshzAS
+BgNVHRMBAf8ECDAGAQH/AgEEMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF
+AAOCAgEAALIY1wkilt/urfEVM5vKzr6utOeDWCUczmWX/RX4ljpRdgF+5fAIS4vH
+tmXkqpSCOVeWUrJV9QvZn6L227ZwuE15cWi8DCDal3Ue90WgAJJZMfTshN4OI8cq
+W9E4EG9wglbEtMnObHlms8F3CHmrw3k6KmUkWGoa+/ENmcVl68u/cMRl1JbW2bM+
+/3A+SAg2c6iPDlehczKx2oa95QW0SkPPWGuNA/CE8CpyANIhu9XFrj3RQ3EqeRcS
+AQQod1RNuHpfETLU/A2gMmvn/w/sx7TB3W5BPs6rprOA37tutPq9u6FTZOcG1Oqj
+C/B7yTqgI7rbyvox7DEXoX7rIiEqyNNUguTk/u3SZ4VXE2kmxdmSh3TQvybfbnXV
+4JbCZVaqiZraqc7oZMnRoWrXRG3ztbnbes/9qhRGI7PqXqeKJBztxRTEVj8ONs1d
+WN5szTwaPIvhkhO3CO5ErU2rVdUr89wKpNXbBODFKRtgxUT70YpmJ46VVaqdAhOZ
+D9EUUn4YaeLaS8AjSF/h7UkjOibNc4qVDiPP+rkehFWM66PVnP1Msh93tc+taIfC
+EYVMxjh8zNbFuoc7fzvvrFILLe7ifvEIUqSVIC/AzplM/Jxw7buXFeGP1qVCBEHq
+391d/9RAfaZ12zkwFsl+IKwE/OZxW8AHa9i1p4GO0YSNuczzEm4=
+-----END CERTIFICATE-----

certs/russian_trusted_sub_ca_pem.crt

@@ -0,0 +1,41 @@
+-----BEGIN CERTIFICATE-----
+MIIHQjCCBSqgAwIBAgICEAIwDQYJKoZIhvcNAQELBQAwcDELMAkGA1UEBhMCUlUx
+PzA9BgNVBAoMNlRoZSBNaW5pc3RyeSBvZiBEaWdpdGFsIERldmVsb3BtZW50IGFu
+ZCBDb21tdW5pY2F0aW9uczEgMB4GA1UEAwwXUnVzc2lhbiBUcnVzdGVkIFJvb3Qg
+Q0EwHhcNMjIwMzAyMTEyNTE5WhcNMjcwMzA2MTEyNTE5WjBvMQswCQYDVQQGEwJS
+VTE/MD0GA1UECgw2VGhlIE1pbmlzdHJ5IG9mIERpZ2l0YWwgRGV2ZWxvcG1lbnQg
+YW5kIENvbW11bmljYXRpb25zMR8wHQYDVQQDDBZSdXNzaWFuIFRydXN0ZWQgU3Vi
+IENBMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA9YPqBKOk19NFymrE
+wehzrhBEgT2atLezpduB24mQ7CiOa/HVpFCDRZzdxqlh8drku408/tTmWzlNH/br
+HuQhZ/miWKOf35lpKzjyBd6TPM23uAfJvEOQ2/dnKGGJbsUo1/udKSvxQwVHpVv3
+S80OlluKfhWPDEXQpgyFqIzPoxIQTLZ0deirZwMVHarZ5u8HqHetRuAtmO2ZDGQn
+vVOJYAjls+Hiueq7Lj7Oce7CQsTwVZeP+XQx28PAaEZ3y6sQEt6rL06ddpSdoTMp
+BnCqTbxW+eWMyjkIn6t9GBtUV45yB1EkHNnj2Ex4GwCiN9T84QQjKSr+8f0psGrZ
+vPbCbQAwNFJjisLixnjlGPLKa5vOmNwIh/LAyUW5DjpkCx004LPDuqPpFsKXNKpa
+L2Dm6uc0x4Jo5m+gUTVORB6hOSzWnWDj2GWfomLzzyjG81DRGFBpco/O93zecsIN
+3SL2Ysjpq1zdoS01CMYxie//9zWvYwzI25/OZigtnpCIrcd2j1Y6dMUFQAzAtHE+
+qsXflSL8HIS+IJEFIQobLlYhHkoE3avgNx5jlu+OLYe0dF0Ykx1PGNjbwqvTX37R
+Cn32NMjlotW2QcGEZhDKj+3urZizp5xdTPZitA+aEjZM/Ni71VOdiOP0igbw6asZ
+2fxdozZ1TnSSYNYvNATwthNmZysCAwEAAaOCAeUwggHhMBIGA1UdEwEB/wQIMAYB
+Af8CAQAwDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBTR4XENCy2BTm6KSo9MI7NM
+XqtpCzAfBgNVHSMEGDAWgBTh0YHlzlpfBKrS6badZrHF+qwshzCBxwYIKwYBBQUH
+AQEEgbowgbcwOwYIKwYBBQUHMAKGL2h0dHA6Ly9yb3N0ZWxlY29tLnJ1L2NkcC9y
+b290Y2Ffc3NsX3JzYTIwMjIuY3J0MDsGCCsGAQUFBzAChi9odHRwOi8vY29tcGFu
+eS5ydC5ydS9jZHAvcm9vdGNhX3NzbF9yc2EyMDIyLmNydDA7BggrBgEFBQcwAoYv
+aHR0cDovL3JlZXN0ci1wa2kucnUvY2RwL3Jvb3RjYV9zc2xfcnNhMjAyMi5jcnQw
+gbAGA1UdHwSBqDCBpTA1oDOgMYYvaHR0cDovL3Jvc3RlbGVjb20ucnUvY2RwL3Jv
+b3RjYV9zc2xfcnNhMjAyMi5jcmwwNaAzoDGGL2h0dHA6Ly9jb21wYW55LnJ0LnJ1
+L2NkcC9yb290Y2Ffc3NsX3JzYTIwMjIuY3JsMDWgM6Axhi9odHRwOi8vcmVlc3Ry
+LXBraS5ydS9jZHAvcm9vdGNhX3NzbF9yc2EyMDIyLmNybDANBgkqhkiG9w0BAQsF
+AAOCAgEARBVzZls79AdiSCpar15dA5Hr/rrT4WbrOfzlpI+xrLeRPrUG6eUWIW4v
+Sui1yx3iqGLCjPcKb+HOTwoRMbI6ytP/ndp3TlYua2advYBEhSvjs+4vDZNwXr/D
+anbwIWdurZmViQRBDFebpkvnIvru/RpWud/5r624Wp8voZMRtj/cm6aI9LtvBfT9
+cfzhOaexI/99c14dyiuk1+6QhdwKaCRTc1mdfNQmnfWNRbfWhWBlK3h4GGE9JK33
+Gk8ZS8DMrkdAh0xby4xAQ/mSWAfWrBmfzlOqGyoB1U47WTOeqNbWkkoAP2ys94+s
+Jg4NTkiDVtXRF6nr6fYi0bSOvOFg0IQrMXO2Y8gyg9ARdPJwKtvWX8VPADCYMiWH
+h4n8bZokIrImVKLDQKHY4jCsND2HHdJfnrdL2YJw1qFskNO4cSNmZydw0Wkgjv9k
+F+KxqrDKlB8MZu2Hclph6v/CZ0fQ9YuE8/lsHZ0Qc2HyiSMnvjgK5fDc3TD4fa8F
+E8gMNurM+kV8PT8LNIM+4Zs+LKEV8nqRWBaxkIVJGekkVKO8xDBOG/aN62AZKHOe
+GcyIdu7yNMMRihGVZCYr8rYiJoKiOzDqOkPkLOPdhtVlgnhowzHDxMHND/E2WA5p
+ZHuNM/m0TXt2wTTPL7JH2YC0gPz/BvvSzjksgzU5rLbRyUKQkgU=
+-----END CERTIFICATE-----

cmd/parser/main.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"log"
+	"net/url"
+	"sync"
+	"time"
+
+	"github.com/terratensor/svodd-server/internal/app"
+	"github.com/terratensor/svodd-server/internal/config"
+	"github.com/terratensor/svodd-server/internal/lib/httpclient"
+	"github.com/terratensor/svodd-server/internal/qaparser/qavideo"
+	"github.com/terratensor/svodd-server/internal/qaparser/questionanswer"
+	"github.com/terratensor/svodd-server/internal/workerpool"
+)
+
+func main() {
+	cfg := config.MustLoad()
+
+	app := app.NewApp(cfg)
+
+	ch := make(chan *url.URL, cfg.EntryChanBuffer)
+
+	wg := &sync.WaitGroup{}
+	for _, parserCfg := range cfg.Parsers {
+
+		wg.Add(1)
+		parser := qavideo.NewParser(parserCfg, *cfg.Delay, *cfg.RandomDelay)
+		go parser.Run(ch, wg)
+	}
+
+	var allTask []*workerpool.Task
+
+	for page := range ch {
+		log.Printf("page: %v", page)
+		task := workerpool.NewTask(func(data interface{}) error {
+			taskID := data.(*url.URL)
+			time.Sleep(100 * time.Millisecond)
+			log.Printf("Task %v processed\n", taskID.String())
+
+			entry := questionanswer.NewEntry(taskID)
+
+			client := httpclient.New(nil)
+			err := entry.FetchData(client)
+			if err != nil {
+				return err
+			}
+
+			return app.Process(entry)
+
+		}, page)
+
+		allTask = append(allTask, task)
+	}
+
+	pool := workerpool.NewPool(allTask, cfg.Workers)
+	pool.Run()
+
+	wg.Wait()
+	log.Println("finished, all tasks successfully processed.")
+}

cmd/service/main.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"log"
+	"net/url"
+	"sync"
+	"time"
+
+	"github.com/terratensor/svodd-server/internal/app"
+	"github.com/terratensor/svodd-server/internal/config"
+	"github.com/terratensor/svodd-server/internal/lib/httpclient"
+	"github.com/terratensor/svodd-server/internal/qaparser/qavideo"
+	"github.com/terratensor/svodd-server/internal/qaparser/questionanswer"
+	"github.com/terratensor/svodd-server/internal/workerpool"
+)
+
+func main() {
+	cfg := config.MustLoad()
+
+	app := app.NewApp(cfg)
+
+	ch := make(chan *url.URL, cfg.EntryChanBuffer)
+
+	wg := &sync.WaitGroup{}
+	for _, parserCfg := range cfg.Parsers {
+
+		wg.Add(1)
+		parser := qavideo.NewParser(parserCfg, *cfg.Delay, *cfg.RandomDelay)
+		go parser.RunBackground(ch, wg)
+	}
+
+	var allTask []*workerpool.Task
+	pool := workerpool.NewPool(allTask, cfg.Workers)
+
+	go func() {
+		for {
+			task := workerpool.NewTask(func(data interface{}) error {
+				taskID := data.(*url.URL)
+				time.Sleep(100 * time.Millisecond)
+				log.Printf("Task %v processed\n", taskID.String())
+
+				entry := questionanswer.NewEntry(taskID)
+
+				client := httpclient.New(nil)
+				err := entry.FetchData(client)
+				if err != nil {
+					return err
+				}
+
+				return app.Process(entry)
+			}, <-ch)
+			pool.AddTask(task)
+		}
+	}()
+
+	pool.RunBackground()
+
+	wg.Wait()
+	log.Println("finished, all workers successfully stopped.")
+}

config/parser/local.yaml

@@ -0,0 +1,16 @@
+env: "local" # Окружение - local, dev или prod
+workers: 1
+delay: 2s
+random_delay: 0s
+manticore:
+  - index: questions
+  - index: questions_ext
+entry_chan_buffer: 20
+splitter:
+  opt_chunk_size: 1800 # оптимальный размер фрагмента контента для поиска, на эти фрагменты будет разбит контент
+  max_chunk_size: 3600 # максимальный размер фрагмента контента для поиска
+parsers:
+  - url: https://xn----8sba0bbi0cdm.xn--p1ai/qa/video
+    current: false
+    pages: 1
+    fetch_all: false

config/parser/prod.yaml

@@ -0,0 +1,16 @@
+env: "prod" # Окружение - local, dev или prod
+workers: 1
+delay: 2s
+random_delay: 0s
+manticore:
+  - index: questions
+  - index: questions_ext
+entry_chan_buffer: 20
+splitter:
+  opt_chunk_size: 1800 # оптимальный размер фрагмента контента для поиска, на эти фрагменты будет разбит контент
+  max_chunk_size: 3600 # максимальный размер фрагмента контента для поиска
+parsers:
+  - url: https://xn----8sba0bbi0cdm.xn--p1ai/qa/video
+    current: false
+    pages: 1
+    fetch_all: false

config/prod.yaml

@@ -0,0 +1,18 @@
+env: "local" # Окружение - local, dev или prod
+workers: 2
+delay: 60s
+random_delay: 150s
+manticore:
+  - index: questions
+  - index: questions_ext
+entry_chan_buffer: 20
+splitter:
+  opt_chunk_size: 1800 # оптимальный размер фрагмента контента для поиска, на эти фрагменты будет разбит контент
+  max_chunk_size: 3600 # максимальный размер фрагмента контента для поиска
+parsers:
+  - url: https://xn----8sba0bbi0cdm.xn--p1ai/qa/video
+    delay: 3600s
+    random_delay: 600s
+    previous: false
+    current: true
+    pages: 1