Skip to content

Commit

Permalink
RF: social tags checker
Browse files Browse the repository at this point in the history
  • Loading branch information
@kynrai
kynrai committed Jun 11, 2024
1 parent 327e8a9 commit 46fd931
Show file tree
Hide file tree
Showing 5 changed files with 159 additions and 176 deletions.
10 changes: 6 additions & 4 deletions checks/checks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,18 @@ import (
)

type Checks struct {
Carbon *Carbon
Rank *Rank
Carbon *Carbon
Rank *Rank
SocialTags *SocialTags
}

func NewChecks() *Checks {
client := &http.Client{
Timeout: 5 * time.Second,
}
return &Checks{
Carbon: NewCarbon(client),
Rank: NewRank(client),
Carbon: NewCarbon(client),
Rank: NewRank(client),
SocialTags: NewSocialTags(client),

Check warning on line 21 in checks/checks.go

View check run for this annotation

Codecov / codecov/patch

checks/checks.go#L19-L21 

Added lines #L19 - L21 were not covered by tests
}
}
94 changes: 94 additions & 0 deletions checks/social_tags.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package checks

import (
"context"
"net/http"

"github.com/PuerkitoBio/goquery"
)

type SocialTagsData struct {
Title string `json:"title"`
Description string `json:"description"`
Keywords string `json:"keywords"`
CanonicalUrl string `json:"canonicalUrl"`
OgTitle string `json:"ogTitle"`
OgType string `json:"ogType"`
OgImage string `json:"ogImage"`
OgUrl string `json:"ogUrl"`
OgDescription string `json:"ogDescription"`
OgSiteName string `json:"ogSiteName"`
TwitterCard string `json:"twitterCard"`
TwitterSite string `json:"twitterSite"`
TwitterCreator string `json:"twitterCreator"`
TwitterTitle string `json:"twitterTitle"`
TwitterDescription string `json:"twitterDescription"`
TwitterImage string `json:"twitterImage"`
ThemeColor string `json:"themeColor"`
Robots string `json:"robots"`
Googlebot string `json:"googlebot"`
Generator string `json:"generator"`
Viewport string `json:"viewport"`
Author string `json:"author"`
Publisher string `json:"publisher"`
Favicon string `json:"favicon"`
}

func (s SocialTagsData) Empty() bool {
return (SocialTagsData{}) == s

Check warning on line 38 in checks/social_tags.go

View check run for this annotation

Codecov / codecov/patch

checks/social_tags.go#L37-L38 

Added lines #L37 - L38 were not covered by tests
}

type SocialTags struct {
client *http.Client
}

func NewSocialTags(client *http.Client) *SocialTags {
return &SocialTags{client: client}

Check warning on line 46 in checks/social_tags.go

View check run for this annotation

Codecov / codecov/patch

checks/social_tags.go#L45-L46 

Added lines #L45 - L46 were not covered by tests
}

func (s *SocialTags) GetSocialTags(ctx context.Context, url string) (*SocialTagsData, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := s.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()

// Parse HTML document
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return nil, err
}

Check warning on line 64 in checks/social_tags.go

View check run for this annotation

Codecov / codecov/patch

checks/social_tags.go#L49-L64 

Added lines #L49 - L64 were not covered by tests

// Extract social tags metadata
tags := &SocialTagsData{
Title: doc.Find("head title").Text(),
Description: doc.Find("meta[name='description']").AttrOr("content", ""),
Keywords: doc.Find("meta[name='keywords']").AttrOr("content", ""),
CanonicalUrl: doc.Find("link[rel='canonical']").AttrOr("href", ""),
OgTitle: doc.Find("meta[property='og:title']").AttrOr("content", ""),
OgType: doc.Find("meta[property='og:type']").AttrOr("content", ""),
OgImage: doc.Find("meta[property='og:image']").AttrOr("content", ""),
OgUrl: doc.Find("meta[property='og:url']").AttrOr("content", ""),
OgDescription: doc.Find("meta[property='og:description']").AttrOr("content", ""),
OgSiteName: doc.Find("meta[property='og:site_name']").AttrOr("content", ""),
TwitterCard: doc.Find("meta[name='twitter:card']").AttrOr("content", ""),
TwitterSite: doc.Find("meta[name='twitter:site']").AttrOr("content", ""),
TwitterCreator: doc.Find("meta[name='twitter:creator']").AttrOr("content", ""),
TwitterTitle: doc.Find("meta[name='twitter:title']").AttrOr("content", ""),
TwitterDescription: doc.Find("meta[name='twitter:description']").AttrOr("content", ""),
TwitterImage: doc.Find("meta[name='twitter:image']").AttrOr("content", ""),
ThemeColor: doc.Find("meta[name='theme-color']").AttrOr("content", ""),
Robots: doc.Find("meta[name='robots']").AttrOr("content", ""),
Googlebot: doc.Find("meta[name='googlebot']").AttrOr("content", ""),
Generator: doc.Find("meta[name='generator']").AttrOr("content", ""),
Viewport: doc.Find("meta[name='viewport']").AttrOr("content", ""),
Author: doc.Find("meta[name='author']").AttrOr("content", ""),
Publisher: doc.Find("link[rel='publisher']").AttrOr("href", ""),
Favicon: doc.Find("link[rel='icon']").AttrOr("href", ""),
}
return tags, nil

Check warning on line 93 in checks/social_tags.go

View check run for this annotation

Codecov / codecov/patch

checks/social_tags.go#L67-L93 

Added lines #L67 - L93 were not covered by tests
}
105 changes: 4 additions & 101 deletions handlers/social_tags.go
View file
Original file line number Diff line number Diff line change
@@ -1,120 +1,23 @@
package handlers

import (
"errors"
"net/http"

"github.com/PuerkitoBio/goquery"
"github.com/xray-web/web-check-api/checks"
)

type SocialTags struct {
Title string `json:"title"`
Description string `json:"description"`
Keywords string `json:"keywords"`
CanonicalUrl string `json:"canonicalUrl"`
OgTitle string `json:"ogTitle"`
OgType string `json:"ogType"`
OgImage string `json:"ogImage"`
OgUrl string `json:"ogUrl"`
OgDescription string `json:"ogDescription"`
OgSiteName string `json:"ogSiteName"`
TwitterCard string `json:"twitterCard"`
TwitterSite string `json:"twitterSite"`
TwitterCreator string `json:"twitterCreator"`
TwitterTitle string `json:"twitterTitle"`
TwitterDescription string `json:"twitterDescription"`
TwitterImage string `json:"twitterImage"`
ThemeColor string `json:"themeColor"`
Robots string `json:"robots"`
Googlebot string `json:"googlebot"`
Generator string `json:"generator"`
Viewport string `json:"viewport"`
Author string `json:"author"`
Publisher string `json:"publisher"`
Favicon string `json:"favicon"`
}

func isEmpty(tags *SocialTags) bool {
return tags.Title == "" &&
tags.Description == "" &&
tags.Keywords == "" &&
tags.CanonicalUrl == "" &&
tags.OgTitle == "" &&
tags.OgType == "" &&
tags.OgImage == "" &&
tags.OgUrl == "" &&
tags.OgDescription == "" &&
tags.OgSiteName == "" &&
tags.TwitterCard == "" &&
tags.TwitterSite == "" &&
tags.TwitterCreator == "" &&
tags.TwitterTitle == "" &&
tags.TwitterDescription == "" &&
tags.TwitterImage == "" &&
tags.ThemeColor == "" &&
tags.Robots == "" &&
tags.Googlebot == "" &&
tags.Generator == "" &&
tags.Viewport == "" &&
tags.Author == "" &&
tags.Publisher == "" &&
tags.Favicon == ""
}

func HandleGetSocialTags() http.Handler {
func HandleGetSocialTags(s *checks.SocialTags) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rawURL, err := extractURL(r)
if err != nil {
JSONError(w, ErrMissingURLParameter, http.StatusBadRequest)
return
}

// Fetch HTML content from the URL
resp, err := http.Get(rawURL.String())
tags, err := s.GetSocialTags(r.Context(), rawURL.String())
if err != nil {
JSONError(w, err, http.StatusInternalServerError)

Check warning on line 18 in handlers/social_tags.go

View check run for this annotation

Codecov / codecov/patch

handlers/social_tags.go#L18 

Added line #L18 was not covered by tests
return
}
defer resp.Body.Close()

// Parse HTML document
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return
}

// Extract social tags metadata
tags := &SocialTags{
Title: doc.Find("head title").Text(),
Description: doc.Find("meta[name='description']").AttrOr("content", ""),
Keywords: doc.Find("meta[name='keywords']").AttrOr("content", ""),
CanonicalUrl: doc.Find("link[rel='canonical']").AttrOr("href", ""),
OgTitle: doc.Find("meta[property='og:title']").AttrOr("content", ""),
OgType: doc.Find("meta[property='og:type']").AttrOr("content", ""),
OgImage: doc.Find("meta[property='og:image']").AttrOr("content", ""),
OgUrl: doc.Find("meta[property='og:url']").AttrOr("content", ""),
OgDescription: doc.Find("meta[property='og:description']").AttrOr("content", ""),
OgSiteName: doc.Find("meta[property='og:site_name']").AttrOr("content", ""),
TwitterCard: doc.Find("meta[name='twitter:card']").AttrOr("content", ""),
TwitterSite: doc.Find("meta[name='twitter:site']").AttrOr("content", ""),
TwitterCreator: doc.Find("meta[name='twitter:creator']").AttrOr("content", ""),
TwitterTitle: doc.Find("meta[name='twitter:title']").AttrOr("content", ""),
TwitterDescription: doc.Find("meta[name='twitter:description']").AttrOr("content", ""),
TwitterImage: doc.Find("meta[name='twitter:image']").AttrOr("content", ""),
ThemeColor: doc.Find("meta[name='theme-color']").AttrOr("content", ""),
Robots: doc.Find("meta[name='robots']").AttrOr("content", ""),
Googlebot: doc.Find("meta[name='googlebot']").AttrOr("content", ""),
Generator: doc.Find("meta[name='generator']").AttrOr("content", ""),
Viewport: doc.Find("meta[name='viewport']").AttrOr("content", ""),
Author: doc.Find("meta[name='author']").AttrOr("content", ""),
Publisher: doc.Find("link[rel='publisher']").AttrOr("href", ""),
Favicon: doc.Find("link[rel='icon']").AttrOr("href", ""),
}

if isEmpty(tags) {
JSONError(w, errors.New("no metadata found"), http.StatusBadRequest)
return
}

JSON(w, tags, http.StatusOK)
})
}
124 changes: 54 additions & 70 deletions handlers/social_tags_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,82 +7,66 @@ import (
"testing"

"github.com/stretchr/testify/assert"
"gopkg.in/h2non/gock.v1"
"github.com/xray-web/web-check-api/checks"
"github.com/xray-web/web-check-api/testutils"
)

func TestHandleGetSocialTags(t *testing.T) {
// t.Parallel()
tests := []struct {
name string
urlParam string
mockResponse string
mockStatusCode int
expectedStatus int
expectedBody map[string]interface{}
}{
{
name: "Missing URL parameter",
urlParam: "",
expectedStatus: http.StatusBadRequest,
expectedBody: map[string]interface{}{"error": "missing URL parameter"},
},
{
name: "Valid URL with social tags",
urlParam: "http://example.com",
mockResponse: `<html><head><title>Example Domain</title><meta name="description" content="Example description"><meta property="og:title" content="Example OG Title"></head><body></body></html>`,
mockStatusCode: http.StatusOK,
expectedStatus: http.StatusOK,
expectedBody: map[string]interface{}{
"title": "Example Domain",
"description": "Example description",
"keywords": "",
"canonicalUrl": "",
"ogTitle": "Example OG Title",
"ogType": "",
"ogImage": "",
"ogUrl": "",
"ogDescription": "",
"ogSiteName": "",
"twitterCard": "",
"twitterSite": "",
"twitterCreator": "",
"twitterTitle": "",
"twitterDescription": "",
"twitterImage": "",
"themeColor": "",
"robots": "",
"googlebot": "",
"generator": "",
"viewport": "",
"author": "",
"publisher": "",
"favicon": "",
},
},
}
t.Parallel()

for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
// t.Parallel()
defer gock.Off()
t.Run("Missing URL parameter", func(t *testing.T) {
t.Parallel()
req := httptest.NewRequest("GET", "/social-tag?url=", nil)
rec := httptest.NewRecorder()

if tc.urlParam != "" {
gock.New(tc.urlParam).
Reply(tc.mockStatusCode).
BodyString(tc.mockResponse)
}
HandleGetSocialTags(checks.NewSocialTags(nil)).ServeHTTP(rec, req)

req := httptest.NewRequest("GET", "/social-tags?url="+tc.urlParam, nil)
rec := httptest.NewRecorder()
HandleGetSocialTags().ServeHTTP(rec, req)
assert.Equal(t, http.StatusBadRequest, rec.Code)
var response KV
err := json.Unmarshal(rec.Body.Bytes(), &response)
assert.NoError(t, err)
assert.Equal(t, KV{"error": "missing URL parameter"}, response)
})

assert.Equal(t, tc.expectedStatus, rec.Code)
t.Run("Valid URL with social tags", func(t *testing.T) {
t.Parallel()

req := httptest.NewRequest("GET", "/social-tags?url=example.com", nil)
rec := httptest.NewRecorder()

HandleGetSocialTags(checks.NewSocialTags(testutils.MockClient(testutils.Response(http.StatusOK, []byte(`<html><head><title>Example Domain</title><meta name="description" content="Example description"><meta property="og:title" content="Example OG Title"></head><body></body></html>`))))).ServeHTTP(rec, req)

assert.Equal(t, http.StatusOK, rec.Code)

var responseBody KV
err := json.Unmarshal(rec.Body.Bytes(), &responseBody)
assert.NoError(t, err)
assert.Equal(t, KV{
"title": "Example Domain",
"description": "Example description",
"keywords": "",
"canonicalUrl": "",
"ogTitle": "Example OG Title",
"ogType": "",
"ogImage": "",
"ogUrl": "",
"ogDescription": "",
"ogSiteName": "",
"twitterCard": "",
"twitterSite": "",
"twitterCreator": "",
"twitterTitle": "",
"twitterDescription": "",
"twitterImage": "",
"themeColor": "",
"robots": "",
"googlebot": "",
"generator": "",
"viewport": "",
"author": "",
"publisher": "",
"favicon": "",
}, responseBody)
})

var responseBody map[string]interface{}
err := json.Unmarshal(rec.Body.Bytes(), &responseBody)
assert.NoError(t, err)
assert.Equal(t, tc.expectedBody, responseBody)
})
}
}
2 changes: 1 addition & 1 deletion server/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func (s *Server) routes() {
s.mux.Handle("GET /api/quality", handlers.HandleGetQuality())
s.mux.Handle("GET /api/rank", handlers.HandleGetRank(s.checks.Rank))
s.mux.Handle("GET /api/redirects", handlers.HandleGetRedirects())
s.mux.Handle("GET /api/social-tags", handlers.HandleGetSocialTags())
s.mux.Handle("GET /api/social-tags", handlers.HandleGetSocialTags(s.checks.SocialTags))

Check warning on line 49 in server/server.go

View check run for this annotation

Codecov / codecov/patch

server/server.go#L49 

Added line #L49 was not covered by tests
s.mux.Handle("GET /api/tls", handlers.HandleTLS())
s.mux.Handle("GET /api/trace-route", handlers.HandleTraceRoute())
}
Expand Down

0 comments on commit 46fd931

Please sign in to comment.