diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e017b2..32a71e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,29 +58,34 @@ jobs: cache: pnpm - run: pnpm install - run: pnpm turbo build - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: 20 - - uses: pnpm/action-setup@v2 - with: - version: 8 - - uses: actions/setup-node@v3 - with: - node-version: 20 - cache: pnpm - - run: pnpm install - - name: Install Playwright Browsers - run: pnpm exec playwright install --with-deps - working-directory: apps/registry - - name: Run Playwright tests - run: pnpm turbo test:e2e --concurrency 1000 # The high concurrency is due to a bug: https://github.com/vercel/turbo/issues/4291 - - uses: actions/upload-artifact@v3 - if: always() - with: - name: playwright-report - path: playwright-report/ - retention-days: 30 + # test: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # - uses: actions/setup-node@v3 + # with: + # node-version: 20 + # - uses: pnpm/action-setup@v2 + # with: + # version: 8 + # - uses: actions/setup-node@v3 + # with: + # node-version: 20 + # cache: pnpm + + # - name: Install dependencies + # run: | + # sudo apt-get update + # sudo apt-get install -y libasound2 libicu-dev libffi-dev libx264-dev liboss4-salsa-asound2 + # - run: pnpm install + # - name: Install Playwright Browsers + # run: pnpm exec playwright install --with-deps + # working-directory: apps/registry + # - name: Run Playwright tests + # run: pnpm turbo test:e2e --concurrency 1000 # The high concurrency is due to a bug: https://github.com/vercel/turbo/issues/4291 + # - uses: actions/upload-artifact@v3 + # if: always() + # with: + # name: playwright-report + # path: playwright-report/ + # retention-days: 30 diff --git a/apps/registry/app/[username]/jobs-graph/JobList.js b/apps/registry/app/[username]/jobs-graph/JobList.js new file mode 100644 index 0000000..4c39e6e --- /dev/null +++ b/apps/registry/app/[username]/jobs-graph/JobList.js @@ -0,0 +1,198 @@ +import React, { useState } from 'react'; +import { + MapPin, + Building, + Calendar, + DollarSign, + BriefcaseIcon, + Globe, + CheckCircle, + Star, +} from 'lucide-react'; +import Link from 'next/link'; + +const JobDescription = ({ job, makeCoverletter }) => { + const [expanded, setExpanded] = useState(false); + console.log({ job }); + return ( +
setExpanded(!expanded)} + > +
+

+ {job.title || 'Not available'} +

+
+
+ + {job.company || 'Not available'} +
+
+ + + {job.salary + ? `$${Number(job.salary).toLocaleString()}/year` + : 'Not available'} + +
+
+ + + {job.location + ? `${job.location.city}, ${job.location.countryCode}` + : 'Not available'} + +
+
+ + {job.experience || 'Not available'} +
+
+ + Remote - {job.remote || 'Not available'} +
+
+ + {job.date || 'Not available'} +
+
+ {expanded ? ( +

{job.description || 'Not available'}

+ ) : ( +

+ {job.description + ? job.description.slice(0, 100) + '...' + : 'Not available'} +

+ )} +
+
+
+ {expanded && ( +
+
+

+ Responsibilities +

+
    + {job.responsibilities?.length ? ( + job.responsibilities.map((resp, index) => ( +
  • + + {resp} +
    • + )) + ) : ( +

    Not available

    + )} +
+
+ +
+

+ Qualifications +

+
    + {job.qualifications?.length ? ( + job.qualifications.map((qual, index) => ( +
  • + + {qual} +
    • + )) + ) : ( +

    Not available

    + )} +
+
+ +
+

+ Skills +

+ {job.skills?.length ? ( + job.skills.map((skill, index) => ( +
+

+ {skill.name} +

+
+ + {skill.level} +
+
+ {skill.keywords?.length ? ( + skill.keywords.map((keyword, kidx) => ( + + {keyword} + + )) + ) : ( +

Not available

+ )} +
+
+ )) + ) : ( +

Not available

+ )} +
+
+ + + View Original Job + + + + View Job Candiates + +
+
+ )} +
+ ); +}; + +const JobList = ({ jobs, makeCoverletter }) => { + const validJobs = jobs?.filter( + (job) => job.gpt_content && job.gpt_content !== 'FAILED' + ); + const fullJobs = validJobs?.map((job) => { + const fullJob = JSON.parse(job.gpt_content); + fullJob.raw = job; + return fullJob; + }); + + return ( +
+ {fullJobs?.map((job, index) => ( + + ))} +
+ ); +}; + +export default JobList; diff --git a/apps/registry/app/[username]/jobs-graph/layout.js b/apps/registry/app/[username]/jobs-graph/layout.js new file mode 100644 index 0000000..4b7c43b --- /dev/null +++ b/apps/registry/app/[username]/jobs-graph/layout.js @@ -0,0 +1,5 @@ +'use client'; + +export default function Home({ children }) { + return <>{children}; +} diff --git a/apps/registry/app/[username]/jobs-graph/page.js b/apps/registry/app/[username]/jobs-graph/page.js new file mode 100644 index 0000000..8cc70a4 --- /dev/null +++ b/apps/registry/app/[username]/jobs-graph/page.js @@ -0,0 +1,692 @@ +'use client'; + +import axios from 'axios'; +import { useEffect, useState, useCallback, useRef, useMemo } from 'react'; +import { forceCollide, forceManyBody } from 'd3-force'; +import ForceGraph2D from 'react-force-graph-2d'; + +// Format skills array into a readable string +const formatSkills = (skills) => { + if (!skills) return ''; + return skills.map((skill) => `${skill.name} (${skill.level})`).join(', '); +}; + +// Format qualifications array into a bullet list +const formatQualifications = (qualifications) => { + if (!qualifications) return ''; + return qualifications.join('\n• '); +}; + +// Helper to format job info into tooltip text +const formatTooltip = (jobInfo) => { + if (!jobInfo) return ''; + + // Truncate description if needed + const truncateText = (text, maxLength) => { + if (!text) return ''; + if (text.length <= maxLength) return text; + return text.substring(0, maxLength - 3) + '...'; + }; + + const parts = [ + `${jobInfo.title || 'Untitled'} at ${jobInfo.company || 'Unknown Company'}`, + jobInfo.remote ? `${jobInfo.remote} Remote` : '', + jobInfo.location && jobInfo.location.city + ? `Location: ${jobInfo.location.city}${ + jobInfo.location.region ? `, ${jobInfo.location.region}` : '' + }` + : '', + `Type: ${jobInfo.type || 'Not specified'}`, + '', + 'Description:', + truncateText(jobInfo.description, 150), + '', + 'Skills:', + formatSkills(jobInfo.skills), + '', + 'Qualifications:', + `• ${formatQualifications(jobInfo.qualifications)}`, + ]; + + return parts.filter(Boolean).join('\n'); +}; + +const calculateCollisionRadius = (node) => { + // Replace this with your logic to determine node size + const nodeSize = node.size || 1; // Default size if not specified + return nodeSize + 3; // Add padding if desired +}; + +export default function Jobs({ params }) { + const { username } = params; + const [jobs, setJobs] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const [isInitialized, setIsInitialized] = useState(false); // Track if initialized + + const [dimensions, setDimensions] = useState({ width: 0, height: 0 }); + const graphRef = useRef(); + const [activeNode, setActiveNode] = useState(null); + const [jobInfo, setJobInfo] = useState({}); // Store parsed job info + const [graphData, setGraphData] = useState(null); + const imageCache = useRef(new Map()); + + const [mostRelevant, setMostRelevant] = useState([]); + const [lessRelevant, setLessRelevant] = useState([]); + + const [readJobs, setReadJobs] = useState(new Set()); + + const [filterText, setFilterText] = useState(''); + const [filteredNodes, setFilteredNodes] = useState(new Set()); + + const [showSalaryGradient, setShowSalaryGradient] = useState(false); + const [salaryRange, setSalaryRange] = useState({ + min: Infinity, + max: -Infinity, + }); + + // Parse salary from various string formats + const parseSalary = useCallback((salary) => { + if (!salary) return null; + if (typeof salary === 'number') return salary; + + const str = salary.toString().toLowerCase(); + // Extract all numbers from the string + const numbers = str.match(/\d+(?:\.\d+)?/g); + if (!numbers) return null; + + // Convert numbers considering k/K multiplier + const values = numbers.map((num) => { + const multiplier = str.includes('k') ? 1000 : 1; + return parseFloat(num) * multiplier; + }); + + // If range, return average + if (values.length > 1) { + values.sort((a, b) => a - b); + return (values[0] + values[values.length - 1]) / 2; + } + + return values[0]; + }, []); + + // Calculate salary range when jobs data changes + useEffect(() => { + if (!jobInfo) return; + + let min = Infinity; + let max = -Infinity; + + Object.values(jobInfo).forEach((job) => { + const salary = parseSalary(job.salary); + if (salary) { + min = Math.min(min, salary); + max = Math.max(max, salary); + } + }); + + if (min !== Infinity && max !== -Infinity) { + setSalaryRange({ min, max }); + } + }, [jobInfo, parseSalary]); + + // Load read jobs from localStorage on mount + useEffect(() => { + const storedReadJobs = localStorage.getItem(`readJobs_${username}`); + if (storedReadJobs) { + setReadJobs(new Set(JSON.parse(storedReadJobs))); + } + }, [username]); + + useEffect(() => { + if (!filterText.trim() || !jobInfo) { + setFilteredNodes(new Set()); + return; + } + + const searchText = filterText.toLowerCase(); + const matches = new Set(); + + Object.entries(jobInfo).forEach(([id, job]) => { + const searchableText = [ + job.title, + job.company, + job.description, + job.type, + job.location?.city, + job.location?.region, + job.skills?.map((s) => s.name).join(' '), + job.qualifications?.join(' '), + ] + .filter(Boolean) + .join(' ') + .toLowerCase(); + + if (searchableText.includes(searchText)) { + matches.add(id); + } + }); + + setFilteredNodes(matches); + }, [filterText, jobInfo]); + + const markJobAsRead = useCallback( + (jobId) => { + setReadJobs((prev) => { + const newReadJobs = new Set(prev); + newReadJobs.add(jobId); + localStorage.setItem( + `readJobs_${username}`, + JSON.stringify([...newReadJobs]) + ); + return newReadJobs; + }); + }, + [username] + ); + + // Memoize node colors for salary view + const nodeSalaryColors = useMemo(() => { + if (!showSalaryGradient || !jobInfo) return new Map(); + + const colors = new Map(); + Object.entries(jobInfo).forEach(([id, job]) => { + const salary = parseSalary(job.salary); + if (salary) { + const percentage = + (salary - salaryRange.min) / (salaryRange.max - salaryRange.min); + const lightBlue = [219, 234, 254]; // bg-blue-100 + const darkBlue = [30, 64, 175]; // bg-blue-800 + + const r = Math.round( + lightBlue[0] + (darkBlue[0] - lightBlue[0]) * percentage + ); + const g = Math.round( + lightBlue[1] + (darkBlue[1] - lightBlue[1]) * percentage + ); + const b = Math.round( + lightBlue[2] + (darkBlue[2] - lightBlue[2]) * percentage + ); + + colors.set(id, `rgb(${r}, ${g}, ${b})`); + } else { + colors.set(id, '#e2e8f0'); // Light gray for no salary + } + }); + return colors; + }, [showSalaryGradient, jobInfo, parseSalary, salaryRange]); + + const getNodeColor = useCallback( + (node) => { + if (node.group === -1) return '#fff'; + if (filterText && !filteredNodes.has(node.id)) return '#f8fafc'; + + if (showSalaryGradient) { + return nodeSalaryColors.get(node.id) || '#e2e8f0'; + } + + return readJobs.has(node.id) ? '#f1f5f9' : '#fef9c3'; + }, + [readJobs, filterText, filteredNodes, showSalaryGradient, nodeSalaryColors] + ); + + const getNodeBackground = useCallback( + (node) => { + if (node.group === -1) return '#fff'; + if (filterText && !filteredNodes.has(node.id)) return '#f8fafc'; + + if (showSalaryGradient && jobInfo[node.id]) { + const salary = parseSalary(jobInfo[node.id].salary); + if (salary) { + const percentage = + (salary - salaryRange.min) / (salaryRange.max - salaryRange.min); + const lightBlue = [219, 234, 254]; // bg-blue-100 + const darkBlue = [30, 64, 175]; // bg-blue-800 + + const r = Math.round( + lightBlue[0] + (darkBlue[0] - lightBlue[0]) * percentage + ); + const g = Math.round( + lightBlue[1] + (darkBlue[1] - lightBlue[1]) * percentage + ); + const b = Math.round( + lightBlue[2] + (darkBlue[2] - lightBlue[2]) * percentage + ); + + return `rgb(${r}, ${g}, ${b})`; + } + return '#e2e8f0'; // Light gray for no salary + } + + return readJobs.has(node.id) ? '#f1f5f9' : '#fef9c3'; + }, + [ + readJobs, + filterText, + filteredNodes, + showSalaryGradient, + jobInfo, + parseSalary, + salaryRange, + ] + ); + + // Function to preload and cache image + const getCachedImage = (src) => { + if (!imageCache.current.has(src)) { + const img = new Image(); + img.src = src; + imageCache.current.set(src, img); + } + return imageCache.current.get(src); + }; + + // Center and zoom the graph when it's ready + const handleEngineStop = useCallback(() => { + console.log('ENGINE STOPPED Graph instance:', graphRef.current); + if (graphRef.current) { + if (!isInitialized) { + const fg = graphRef.current; + console.log('FG IS STARTING', fg); + if (fg) { + // Deactivate existing forces if necessary + // fg.d3Force('center', null); + fg.d3Force( + 'charge', + forceManyBody() + .strength(-300) // Negative values repel nodes; adjust this value for more/less repulsion + .distanceMax(600) // Maximum distance where the charge force is applied + .distanceMin(20) // Minimum distance where the charge force is applied + ); + + // Add custom collision force + fg.d3Force( + 'collide', + forceCollide().radius((node) => calculateCollisionRadius(node)) + ); + + setIsInitialized(true); + } + } + } + }, [isInitialized]); + + const handleCanvasClick = useCallback( + (event) => { + if (!graphRef.current) return; + + const canvas = graphRef.current.canvas; + const rect = canvas.getBoundingClientRect(); + const x = event.clientX - rect.left; + const y = event.clientY - rect.top; + + // Check if click is on a node + const clickedNode = graphData?.nodes.find((node) => { + const dx = x - node.x; + const dy = y - node.y; + return Math.sqrt(dx * dx + dy * dy) <= node.size; + }); + + if (clickedNode) { + setActiveNode(clickedNode); + } + }, + [graphData] + ); + + useEffect(() => { + if (graphRef.current?.canvas && dimensions.width && dimensions.height) { + const canvas = graphRef.current.canvas; + canvas.addEventListener('click', handleCanvasClick); + + return () => { + canvas.removeEventListener('click', handleCanvasClick); + }; + } + }, [handleCanvasClick, dimensions]); + + useEffect(() => { + const container = document.getElementById('graph-container'); + if (container) { + const width = container.offsetWidth; + const height = 600; + setDimensions({ width, height }); + } + }, []); + + useEffect(() => { + const fetchData = async () => { + setIsLoading(true); + try { + const response = await axios.post('/api/jobs-graph', { username }); + const { graphData, jobInfoMap, mostRelevant, lessRelevant, allJobs } = + response.data; + + setMostRelevant(mostRelevant); + setLessRelevant(lessRelevant); + setJobs(allJobs); + setJobInfo(jobInfoMap); + setGraphData(graphData); + } catch (error) { + console.error('Error fetching data:', error); + } finally { + setIsLoading(false); + } + }; + + if (username) { + fetchData(); + } + }, [username]); + + if (isLoading || !graphData) { + return ( +
+
+

Loading jobs graph...

+
+
+
+ ); + } + + return ( +
+ {/* */} + + {/* {!jobs && } */} +
+ {jobs ? ( +
+

+ Found {jobs.length} related jobs ({mostRelevant.length} highly + relevant) +

+
+ setFilterText(e.target.value)} + className="flex-1 px-4 py-2 border border-gray-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500" + /> +
+ +
+
+
+ ) : ( +

Loading jobs...

+ )} +
+ +
+ {activeNode && ( +
+
+
+ + +
+ {formatTooltip(jobInfo[activeNode.id])} +
+
+

+ {jobInfo[activeNode.id]?.location?.city || 'Remote'}{' '} + {jobInfo[activeNode.id]?.type || ''} +

+
+ e.stopPropagation()} + > + View Details + +
+ {jobInfo[activeNode.id]?.salary && ( +

+ Salary: {jobInfo[activeNode.id].salary} +

+ )} +
+
+ )} + getNodeColor(node)} + nodeVal={(node) => node.size} + nodeCanvasObjectMode={() => 'after'} + nodeCanvasObject={(node, ctx) => { + // Set resume node size + if (node.group === -1) { + node.size = 80; + } + // Calculate other node sizes based on relevance + else { + const jobIndex = [...mostRelevant, ...lessRelevant].findIndex( + (j) => j.uuid === node.id + ); + if (jobIndex !== -1) { + const maxSize = 36; + const minSize = 4; + const sizeRange = maxSize - minSize; + const totalJobs = mostRelevant.length + lessRelevant.length; + node.size = Math.max( + minSize, + maxSize - (sizeRange * jobIndex) / totalJobs + ); + } + } + + // Draw node + ctx.beginPath(); + ctx.arc(node.x, node.y, node.size, 0, 2 * Math.PI); + ctx.fillStyle = getNodeBackground(node); + ctx.fill(); + ctx.strokeStyle = getNodeColor(node); + ctx.lineWidth = 2; + ctx.stroke(); + + if (node.group === -1 && node.image) { + // Resume node with image + const img = getCachedImage(node.image); + + if (img.complete) { + ctx.save(); + ctx.beginPath(); + ctx.arc(node.x, node.y, node.size, 0, 2 * Math.PI); + ctx.clip(); + ctx.drawImage( + img, + node.x - node.size, + node.y - node.size, + node.size * 2, + node.size * 2 + ); + ctx.strokeStyle = '#000'; + ctx.lineWidth = 2; + ctx.stroke(); + ctx.restore(); + } else { + // Draw default circle while image is loading + ctx.beginPath(); + ctx.arc(node.x, node.y, node.size, 0, 2 * Math.PI); + ctx.fillStyle = getNodeColor(node); + ctx.fill(); + ctx.strokeStyle = '#000'; + ctx.lineWidth = 2; + ctx.stroke(); + } + } else { + // Default node rendering for all other cases + ctx.beginPath(); + ctx.arc(node.x, node.y, node.size, 0, 2 * Math.PI); + ctx.fillStyle = getNodeColor(node); + ctx.fill(); + ctx.strokeStyle = '#000'; + ctx.lineWidth = 1; + ctx.stroke(); + } + + // Draw rank number for job nodes + if (node.group !== -1) { + const jobIndex = [...mostRelevant, ...lessRelevant].findIndex( + (j) => j.uuid === node.id + ); + if (jobIndex !== -1) { + const fontSize = Math.max(10, node.size * 0.8); + ctx.font = `${fontSize}px Sans-Serif`; + ctx.textAlign = 'center'; + ctx.textBaseline = 'middle'; + ctx.fillStyle = '#000'; + ctx.fillText(jobIndex + 1, node.x, node.y); + } + } + + // Draw regular label for resume node + if (node.group === -1) { + const label = node.label || node.id; + const fontSize = Math.max(14, node.size); + ctx.font = `bold ${fontSize}px Sans-Serif`; + const textWidth = ctx.measureText(label).width; + const bckgDimensions = [textWidth, fontSize].map( + (n) => n + fontSize * 0.2 + ); + + // Draw background for label + ctx.fillStyle = 'rgba(255, 255, 255, 0.9)'; + ctx.strokeStyle = '#000'; + ctx.lineWidth = 1; + ctx.beginPath(); + ctx.roundRect( + node.x - bckgDimensions[0] / 2, + node.y - bckgDimensions[1] * 2, + bckgDimensions[0], + bckgDimensions[1], + 5 + ); + ctx.fill(); + ctx.stroke(); + + // Draw label + ctx.textAlign = 'center'; + ctx.textBaseline = 'middle'; + ctx.fillStyle = '#000'; + ctx.fillText(label, node.x, node.y - bckgDimensions[1] * 1.5); + } + }} + nodePointerAreaPaint={(node, color, ctx) => { + // Draw a larger hit area for hover detection + ctx.beginPath(); + ctx.arc(node.x, node.y, node.size * 2, 0, 2 * Math.PI); + ctx.fillStyle = color; + ctx.fill(); + }} + onRenderFramePost={() => { + // No need to render tooltip in canvas anymore since we're using DOM + }} + linkWidth={(link) => Math.sqrt(link.value) * 2} + linkColor="#cccccc" + linkOpacity={0.3} + enableNodeDrag={true} + cooldownTicks={100} + warmupTicks={100} + width={dimensions.width} + height={dimensions.height} + onEngineStop={handleEngineStop} + minZoom={0.1} + maxZoom={5} + forceEngine="d3" + d3AlphaDecay={0.02} + d3VelocityDecay={0.3} + onNodeHover={(node) => { + if (node) { + setActiveNode(node); + } + }} + /> +
+
+ ); +} diff --git a/apps/registry/app/api/similarity/route.js b/apps/registry/app/api/similarity/route.js index 395c723..141d3d0 100644 --- a/apps/registry/app/api/similarity/route.js +++ b/apps/registry/app/api/similarity/route.js @@ -21,10 +21,23 @@ export async function GET(request) { const limit = parseInt(searchParams.get('limit')) || 1000; console.time('getResumeSimilarityData'); - const { data, error } = await supabase + // First fetch thomasdavis's resume + const { data: thomasData, error: thomasError } = await supabase + .from('resumes') + .select('username, embedding, resume') + .eq('username', 'thomasdavis') + .single(); + + if (thomasError) { + console.error('Error fetching thomasdavis resume:', thomasError); + } + + // Then fetch other resumes + const { data: otherData, error } = await supabase .from('resumes') .select('username, embedding, resume') .not('embedding', 'is', null) + .neq('username', 'thomasdavis') // Exclude thomasdavis from this query .limit(limit) .order('created_at', { ascending: false }); @@ -38,6 +51,9 @@ export async function GET(request) { console.timeEnd('getResumeSimilarityData'); + // Combine the results, putting thomasdavis first if available + const data = thomasData ? [thomasData, ...(otherData || [])] : otherData; + // Parse embeddings from strings to numerical arrays and extract position const parsedData = data .map((item) => { diff --git a/apps/registry/pages/api/jobs-graph.js b/apps/registry/pages/api/jobs-graph.js new file mode 100644 index 0000000..bc2919d --- /dev/null +++ b/apps/registry/pages/api/jobs-graph.js @@ -0,0 +1,184 @@ +const OpenAI = require('openai'); +const { createClient } = require('@supabase/supabase-js'); + +const supabaseUrl = 'https://itxuhvvwryeuzuyihpkp.supabase.co'; +const supabaseKey = process.env.SUPABASE_KEY; +const supabase = createClient(supabaseUrl, supabaseKey); + +const cosineSimilarity = (a, b) => { + const dotProduct = a.reduce((sum, val, i) => sum + val * b[i], 0); + const magnitudeA = Math.sqrt(a.reduce((sum, val) => sum + val * val, 0)); + const magnitudeB = Math.sqrt(b.reduce((sum, val) => sum + val * val, 0)); + return dotProduct / (magnitudeA * magnitudeB); +}; + +export default async function handler(req, res) { + if (!process.env.OPENAI_API_KEY) { + throw new Error('Missing env var from OpenAI'); + } + + const openai = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, + }); + + const { username } = req.body; + + const { data } = await supabase + .from('resumes') + .select() + .eq('username', username); + + const resume = JSON.parse(data[0].resume); + const resumeCompletion = await openai.chat.completions.create({ + model: 'gpt-3.5-turbo-16k', + messages: [ + { + role: 'system', + content: + "You are a professional resume analyzer. Create a detailed professional summary that describes this candidate's background, skills, and experience in natural language. Focus on their expertise, achievements, and what makes them unique. Write it in a style similar to job descriptions to optimize for semantic matching. Do not include the candidates name. Make sure to include everything significant to the users career. Describe the type of industries they have experience in.", + }, + { + role: 'user', + content: JSON.stringify(resume), + }, + ], + temperature: 0.85, + }); + + const resumeDescription = resumeCompletion.choices[0].message.content; + + // const resumeDescription = ` + // Professional Summary + // Dynamic and accomplished Full-Stack Web Developer with extensive experience in building scalable, user-focused applications from the ground up, particularly in startup environments. Adept at wearing multiple hats to deliver robust, product-driven solutions that prioritize user feedback and high-impact results. A recognized leader in the open-source community and a trusted contributor to several high-profile initiatives, with a proven track record of driving innovation and collaboration. + + // Key Expertise & Skills + // Frontend Development: Advanced skills in HTML, SCSS/CSS (BEM, Styled Components), JavaScript/TypeScript, React, Next.js, Redux, and Apollo. + // Backend Development: Expertise in Node.js, Ruby, Python, PostgreSQL, Redis, and serverless architectures. + // DevOps: Proficient in AWS, Google Cloud, Heroku, and caching strategies with Fastly and Cloudflare. Experienced in Docker and Kubernetes, holding a Certified Kubernetes Administrator credential. + // Leadership: Demonstrated success in team management, including CTO-level responsibilities, with a history of scaling startups and rescuing critical projects under tight deadlines. + // Open-Source Advocacy: Founder of initiatives like JSON Resume and Cdnjs, serving millions of developers and websites globally, with a strong commitment to fostering community-driven solutions.`; + + const completion = await openai.embeddings.create({ + model: 'text-embedding-3-large', + input: resumeDescription, + }); + + const desiredLength = 3072; + let embedding = completion.data[0].embedding; + if (embedding.length < desiredLength) { + embedding = embedding.concat( + Array(desiredLength - embedding.length).fill(0) + ); + } + + const { data: documents } = await supabase.rpc('match_jobs_v5', { + query_embedding: embedding, + match_threshold: -1, + match_count: 150, + }); + + const sortedDocuments = documents.sort((a, b) => b.similarity - a.similarity); + const jobIds = documents ? sortedDocuments.map((doc) => doc.id) : []; + + const { data: jobsData } = await supabase + .from('jobs') + .select('*') + .in('id', jobIds) + .order('created_at', { ascending: false }); + + // Add similarity scores to jobs + const sortedJobs = jobsData.map((job) => { + const doc = sortedDocuments.find((d) => d.id === job.id); + return { + ...job, + similarity: doc ? doc.similarity : 0, + }; + }); + + // Split into most relevant (top 20) and less relevant + const topJobs = sortedJobs.slice(0, 20); + const otherJobs = sortedJobs.slice(20); + + // Create graph data + const graphData = { + nodes: [ + { + id: username, + group: -1, + size: 24, + color: resume.basics?.image ? 'url(#resumeImage)' : '#ff0000', + x: 0, + y: 0, + image: resume.basics?.image || null, + }, + ...topJobs.map((job) => ({ + id: job.uuid, + label: JSON.parse(job.gpt_content).title, + group: 1, + size: 4, + color: '#fff18f', + vector: JSON.parse(job.embedding_v5), + })), + ...otherJobs.map((job) => ({ + id: job.uuid, + label: JSON.parse(job.gpt_content).title, + group: 2, + size: 4, + color: '#fff18f', + vector: JSON.parse(job.embedding_v5), + })), + ], + links: [ + ...topJobs.map((job) => ({ + source: username, + target: job.uuid, + value: job.similarity, + })), + // Process other jobs sequentially, each one only looking at previously processed jobs + ...otherJobs.reduce((links, lessRelevantJob, index) => { + const lessRelevantVector = JSON.parse(lessRelevantJob.embedding_v5); + + // Jobs to compare against: top jobs + already processed less relevant jobs + const availableJobs = [...topJobs, ...otherJobs.slice(0, index)]; + + const mostSimilarJob = availableJobs.reduce( + (best, current) => { + const similarity = cosineSimilarity( + lessRelevantVector, + JSON.parse(current.embedding_v5) + ); + return similarity > best.similarity + ? { job: current, similarity } + : best; + }, + { job: null, similarity: -1 } + ); + + if (mostSimilarJob.job) { + links.push({ + source: mostSimilarJob.job.uuid, + target: lessRelevantJob.uuid, + value: mostSimilarJob.similarity, + }); + } + + return links; + }, []), + ], + }; + + // Create job info map + const jobInfoMap = {}; + sortedJobs.forEach((job) => { + jobInfoMap[job.uuid] = JSON.parse(job.gpt_content); + }); + + res.status(200).json({ + graphData, + jobInfoMap, + mostRelevant: topJobs, + lessRelevant: otherJobs, + allJobs: sortedJobs, + resume: resume, + }); +} diff --git a/apps/registry/pages/api/jobs.js b/apps/registry/pages/api/jobs.js index 6817e38..a52d2cc 100644 --- a/apps/registry/pages/api/jobs.js +++ b/apps/registry/pages/api/jobs.js @@ -30,7 +30,7 @@ export default async function handler(req, res) { { role: 'system', content: - "You are a professional resume analyzer. Create a detailed professional summary that describes this candidate's background, skills, and experience in natural language. Focus on their expertise, achievements, and what makes them unique. Write it in a style similar to job descriptions to optimize for semantic matching. Do not include the candidates name. Be terse as most job descriptions are.", + "You are a professional resume analyzer. Create a detailed professional summary that describes this candidate's background, skills, and experience in natural language. Focus on their expertise, achievements, and what makes them unique. Write it in a style similar to job descriptions to optimize for semantic matching. Do not include the candidates name. Make sure to include everything significant to the users career. Describe the type of industries they have experience in.", }, { role: 'user', @@ -41,7 +41,6 @@ export default async function handler(req, res) { }); const resumeDescription = resumeCompletion.choices[0].message.content; - console.log({ resumeDescription }); const completion = await openai.embeddings.create({ @@ -61,29 +60,68 @@ export default async function handler(req, res) { const { data: documents } = await supabase.rpc('match_jobs_v5', { query_embedding: embedding, - match_threshold: 0.02, // Choose an appropriate threshold for your data - match_count: 200, // Choose the number of matches + match_threshold: -1, // Choose an appropriate threshold for your data + match_count: 500, // Choose the number of matches }); - console.log({ documents }); + // Log initial match count + console.log('Total matched jobs:', documents.length); + // similarity is on documents, it is a flow, i want to sort from highest to lowest - // then get the job ids const sortedDocuments = documents.sort((a, b) => b.similarity - a.similarity); const jobIds = documents ? sortedDocuments.map((doc) => doc.id) : []; - const { data: jobs } = await supabase.from('jobs').select().in('id', jobIds); - // sort jobs in the same order as jobIds by id and add similarity scores - const sortedJobs = jobIds.map((id, index) => { - const job = jobs.find((job) => job.id === id); + const { data: sortedJobs } = await supabase + .from('jobs') + .select('*') + .in('id', jobIds) + .order('created_at', { ascending: false }); + + // Add similarity scores to jobs + const jobsWithSimilarity = sortedJobs.map((job) => { + const doc = sortedDocuments.find((d) => d.id === job.id); return { ...job, - similarity: documents[index].similarity, + similarity: doc ? doc.similarity : 0, }; }); - const filteredJobs = sortedJobs.filter( + // Get date distribution + const validDates = jobsWithSimilarity + .map((job) => { + const date = new Date(job.created_at); + // Check if date is valid + if (isNaN(date.getTime())) { + console.log('Invalid date found:', { + id: job.id, + created_at: job.created_at, + }); + return null; + } + return date; + }) + .filter(Boolean); // Remove null values + + if (validDates.length > 0) { + const oldestDate = new Date(Math.min(...validDates)); + const newestDate = new Date(Math.max(...validDates)); + + console.log('Date range of jobs:', { + oldest: oldestDate.toISOString(), + newest: newestDate.toISOString(), + daysBetween: Math.floor( + (newestDate - oldestDate) / (1000 * 60 * 60 * 24) + ), + validDatesCount: validDates.length, + invalidDatesCount: jobsWithSimilarity.length - validDates.length, + }); + } else { + console.log('No valid dates found in the jobs'); + } + + const filteredJobs = jobsWithSimilarity.filter( (job) => - new Date(job.created_at) > new Date(Date.now() - 60 * 24 * 60 * 90 * 1000) + new Date(job.created_at) > new Date(Date.now() - 60 * 24 * 60 * 60 * 1000) ); return res.status(200).send(filteredJobs); diff --git a/apps/registry/scripts/jobs/hackernews.js b/apps/registry/scripts/jobs/hackernews.js index fafd2e3..fb84d89 100644 --- a/apps/registry/scripts/jobs/hackernews.js +++ b/apps/registry/scripts/jobs/hackernews.js @@ -8,7 +8,7 @@ const supabaseKey = process.env.SUPABASE_KEY; const supabase = createClient(supabaseUrl, supabaseKey); const HN_API = 'https://hn.algolia.com/api/v1/items/'; -const WHO_IS_HIRING_ITEM_ID = 42297424; +const WHO_IS_HIRING_ITEM_ID = 42575537; async function main() { const response = await axios.get(`${HN_API}${WHO_IS_HIRING_ITEM_ID}`);