Skip to content

Commit

Permalink
fix: gracefully handle timeout in chromedp query when no nodes are re…
Browse files Browse the repository at this point in the history
…turned
  • Loading branch information
b-dulaney committed Mar 25, 2024
1 parent 384b9ef commit f297acc
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 54 deletions.
67 changes: 16 additions & 51 deletions scraping.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"time"

"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/page"
"github.com/chromedp/chromedp"
"golang.org/x/text/cases"
"golang.org/x/text/language"
Expand All @@ -20,14 +19,24 @@ func clickProvidedSelector(ctx context.Context, config Config) {
}

func countLiftsAndRuns(ctx context.Context, config Config) (int, int) {
tctx, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
var openLifts, openRuns []*cdp.Node
if config.Terrain.RunClickInteraction {
var buttonsToClick []*cdp.Node
runChromeDP(ctx,
err := runChromeDP(tctx,
chromedp.WaitReady(config.Terrain.LiftsOpenSelector),
chromedp.Nodes(config.Terrain.LiftStatusSelector, &openLifts, chromedp.ByQueryAll),
chromedp.Nodes(config.Terrain.RunClickSelector, &buttonsToClick, chromedp.ByQueryAll),
)

if err != nil {
log.Printf("No open lift nodes found")
runChromeDP(ctx,
chromedp.Nodes(config.Terrain.RunClickSelector, &buttonsToClick, chromedp.ByQueryAll),
)
}

if len(buttonsToClick) == 0 {
log.Panic("No buttons provided to click")
}
Expand Down Expand Up @@ -76,51 +85,7 @@ func processTerrain(ctx context.Context, config Config, terrainNodes []*cdp.Node
}

func navigateToURL(ctx context.Context, url string) {
const script = `(function(w, n, wn) {
// Pass the Webdriver Test.
Object.defineProperty(n, 'webdriver', {
get: () => false,
});
// Pass the Plugins Length Test.
// Overwrite the plugins property to use a custom getter.
Object.defineProperty(n, 'plugins', {
// This just needs to have length > 0 for the current test,
// but we could mock the plugins too if necessary.
get: () => [1, 2, 3, 4, 5],
});
// Pass the Languages Test.
// Overwrite the plugins property to use a custom getter.
Object.defineProperty(n, 'languages', {
get: () => ['en-US', 'en'],
});
// Pass the Chrome Test.
// We can mock this in as much depth as we need for the test.
w.chrome = {
runtime: {},
};
// Pass the Permissions Test.
const originalQuery = wn.permissions.query;
return wn.permissions.query = (parameters) => (
parameters.name === 'notifications' ?
Promise.resolve({ state: Notification.permission }) :
originalQuery(parameters)
);
})(window, navigator, window.navigator);`

runChromeDP(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
var err error
_, err = page.AddScriptToEvaluateOnNewDocument(script).Do(ctx)
if err != nil {
return err
}
return nil
}),
chromedp.EmulateViewport(1200, 1000), chromedp.Navigate(url))
runChromeDP(ctx, chromedp.EmulateViewport(1200, 1000), chromedp.Navigate(url))
log.Printf("Visiting %s", url)
}

Expand Down Expand Up @@ -148,7 +113,7 @@ func getTerrainData(ctx context.Context, config Config) (runsOpen, liftsOpen int
clickProvidedSelector(ctx, config)
terrainNodes := getTerrainNodes(ctx, config)
liftsOpen, runsOpen = processTerrain(ctx, config, terrainNodes)
return liftsOpen, runsOpen
return runsOpen, liftsOpen
}

// Handles cases where the terrain data is on a separate page
Expand All @@ -159,12 +124,12 @@ func getTerrainData(ctx context.Context, config Config) (runsOpen, liftsOpen int
// and we need to count them ourselves
if config.Terrain.CountLifts {
liftsOpen, runsOpen = countLiftsAndRuns(ctx, config)
return liftsOpen, runsOpen
return runsOpen, liftsOpen
}

terrainNodes := getTerrainNodes(ctx, config)
liftsOpen, runsOpen = processTerrain(ctx, config, terrainNodes)
return liftsOpen, runsOpen
return runsOpen, liftsOpen
}

// Handles cases where the terrain data is on the same page as the conditions data
Expand Down Expand Up @@ -197,7 +162,7 @@ func scrapeResortData(configPath *string) (success bool) {
"updated_at": time.Now(),
}

ctx, cancel := chromedp.NewContext(context.Background(), chromedp.WithDebugf(log.Printf))
ctx, cancel := chromedp.NewContext(context.Background(), chromedp.WithLogf(log.Printf))

defer cancel()

Expand Down
8 changes: 5 additions & 3 deletions utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,12 @@ func initializeSupabase() *supabase.Client {
return client
}

func runChromeDP(ctx context.Context, tasks ...chromedp.Action) {
if err := chromedp.Run(ctx, tasks...); err != nil {
log.Fatalf("Failed running chromedp tasks: %s", err)
func runChromeDP(ctx context.Context, tasks ...chromedp.Action) error {
err := chromedp.Run(ctx, tasks...)
if err != nil {
return err
}
return nil
}

func getTextFromNode(ctx context.Context, selector string, node *cdp.Node, result *string) {
Expand Down

0 comments on commit f297acc

Please sign in to comment.