Skip to content

Commit

Permalink
Fix Replicate loader
Browse files Browse the repository at this point in the history
  • Loading branch information
andreasjansson committed May 23, 2024
1 parent fa9a439 commit ea42354
Showing 1 changed file with 18 additions and 15 deletions.
33 changes: 18 additions & 15 deletions lib/content.js
Original file line number Diff line number Diff line change
Expand Up @@ -30,41 +30,35 @@ async function fetchReplicatePosts() {
const oneWeekAgo = new Date();
oneWeekAgo.setDate(oneWeekAgo.getDate() - 7);

loop:
for await (const batch of replicate.paginate(replicate.models.list)) {
if (posts.length >= limit) break;

for (const model of batch) {
if (model.latest_version == null || model.latest_version.id == null || model.run_count <= 1) continue;

if (new Date(model.latest_version.created_at) < oneWeekAgo) break;

const versionResponse = await replicate.models.versions.list(
model.owner,
model.name
);

const versions = versionResponse.results;
const firstVersion = versions[versions.length - 1];
if (new Date(model.latest_version.created_at) < oneWeekAgo) {
break loop;
}

const smallerId = model.latest_version.id.slice(0, 12);
const repoIdInt = BigInteger.parse(smallerId, 36).toString();
const modelHashInt = hashStringToInt(model.url).toString();

const post = new Post(
repoIdInt,
modelHashInt,
"replicate",
model.owner,
model.name,
model.run_count,
model.description,
model.url,
firstVersion.created_at
model.created_at
);
posts.push(post);
}

console.log(`Uploaded ${posts.length} replicate models`); // Log after processing the batch
}

console.log(`Uploaded ${posts.length} replicate models`); // Log after processing the batch

return posts;
}

Expand Down Expand Up @@ -348,3 +342,12 @@ function truncateWithoutBreakingWords(str, n) {

return truncatedStr.substr(0, lastSpaceIndex) + "...";
}

function hashStringToInt(str) {
let hash = 0;
for (let i = 0; i < str.length; i++) {
hash = (hash << 5) - hash + str.charCodeAt(i);
hash |= 0; // Convert to 32bit integer
}
return hash;
}

0 comments on commit ea42354

Please sign in to comment.