utils/shl_product_scraper.py

import requests
from bs4 import BeautifulSoup
import csv
import re
import time
from urllib.parse import urljoin

def scrape_shl_products():
    # URL to scrape
    base_url = "https://www.shl.com"
    catalog_url = "https://www.shl.com/solutions/products/product-catalog/"

    # Send HTTP request with improved headers
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.5',
        'Connection': 'keep-alive',
        'Upgrade-Insecure-Requests': '1',
        'Cache-Control': 'max-age=0'
    }

    try:
        # First try to get the main product catalog page
        print(f"Fetching main catalog: {catalog_url}")
        response = requests.get(catalog_url, headers=headers)
        response.raise_for_status()

        soup = BeautifulSoup(response.content, 'html.parser')

        # Look for the actual product catalog items
        view_links = []

        for link in soup.find_all('a', href=True):
            href = link['href']
            if '/product-catalog/view/' in href:
                view_links.append(href)

        print(f"Found {len(view_links)} product catalog view links")

        # Get more detail pages by looking at pagination
        pagination_links = []
        for link in soup.find_all('a', href=True):
            href = link['href']
            if '/product-catalog/' in href and ('?start=' in href or '&start=' in href):
                pagination_links.append(href)

        if pagination_links:
            print(f"Found {len(pagination_links)} pagination links")

            # Process each pagination page
            for page_url in pagination_links:
                # Make sure the URL is absolute
                if not page_url.startswith('http'):
                    page_url = urljoin(base_url, page_url)

                print(f"Fetching pagination page: {page_url}")
                try:
                    page_response = requests.get(page_url, headers=headers)
                    page_response.raise_for_status()
                    page_soup = BeautifulSoup(page_response.content, 'html.parser')

                    # Find all product view links on this pagination page
                    for link in page_soup.find_all('a', href=True):
                        href = link['href']
                        if '/product-catalog/view/' in href:
                            view_links.append(href)

                except Exception as e:
                    print(f"Error fetching pagination page: {e}")

                # Be nice to the server
                time.sleep(1)

        # Remove duplicates and ensure all links are absolute
        view_links = list(set(view_links))
        view_links = [urljoin(base_url, link) if not link.startswith('http') else link for link in view_links]

        print(f"Found {len(view_links)} unique product view links")

        # Now we'll scrape each individual product page
        products = []

        for i, product_url in enumerate(view_links):
            print(f"Scraping product {i+1}/{len(view_links)}: {product_url}")

            try:
                # Delay between requests to be polite to the server
                if i > 0:
                    time.sleep(1)

                product_response = requests.get(product_url, headers=headers)
                product_response.raise_for_status()
                product_soup = BeautifulSoup(product_response.content, 'html.parser')

                # Extract product name - usually in the title or main heading
                product_name = ""
                # Try to get it from the title
                title_tag = product_soup.find('title')
                if title_tag:
                    title_text = title_tag.text.strip()
                    # Clean up title - often in format "Product Name | SHL"
                    if '|' in title_text:
                        product_name = title_text.split('|')[0].strip()

                # If no name from title, try the H1
                if not product_name:
                    h1_tag = product_soup.find('h1')
                    if h1_tag:
                        product_name = h1_tag.text.strip()

                # If still no name, extract from URL
                if not product_name:
                    url_parts = product_url.rstrip('/').split('/')
                    product_name = url_parts[-1].replace('-', ' ').title()

                # Get page content as text for analysis
                page_text = product_soup.get_text().lower()

                # Try to determine if remote testing is supported
                remote_testing = "Unknown"
                remote_terms = ['remote', 'online', 'virtual', 'internet', 'web-based', 'digital', 'web browser',
                               'online platform', 'from anywhere', 'off-site', 'distance']
                remote_phrases = [
                    'take the test remotely', 'administer remotely', 'online assessment', 'digital delivery',
                    'web-based platform', 'remote proctoring', 'internet connection', 'browser-based',
                    'accessible anywhere', 'remote testing', 'online testing'
                ]

                # Check for remote testing keywords
                for term in remote_terms:
                    if term in page_text:
                        remote_testing = "Yes"
                        break

                # If not found with simple terms, check for phrases
                if remote_testing == "Unknown":
                    for phrase in remote_phrases:
                        if phrase in page_text:
                            remote_testing = "Yes"
                            break

                # Most modern SHL tests are remote, so if we're still uncertain, check for contrary evidence
                if remote_testing == "Unknown" and not any(x in page_text for x in ['in-person only', 'on-site only', 'physical test center required']):
                    # If product URL or name contains certain keywords, it's likely remote
                    product_url_lower = product_url.lower()
                    if any(x in product_url_lower for x in ['online', 'digital', 'remote', 'virtual']):
                        remote_testing = "Yes"
                    # For SHL products, most are remote unless explicitly stated otherwise
                    elif 'shl.com' in product_url:
                        remote_testing = "Yes"

                # ENHANCED ADAPTIVE/IRT DETECTION
                adaptive = "Unknown"
                # Direct adaptive terminology
                adaptive_terms = [
                    'adaptive', 'irt', 'item response theory', 'tailored', 'adjusts difficulty',
                    'computer adaptive', 'cat', 'adaptive testing', 'dynamic difficulty',
                    'adjusts questions', 'personalized assessment', 'adaptive algorithm',
                    'smart testing', 'tailored questioning', 'adaptive assessment'
                ]

                # Phrases that indicate adaptive testing
                adaptive_phrases = [
                    'questions adapt based on', 'difficulty adjusts', 'tailored to ability',
                    'adapts to the test taker', 'customizes questions', 'dynamic question selection',
                    'questions change based on previous answers', 'adapts to user performance',
                    'intelligent testing algorithm', 'questions get harder or easier'
                ]

                # Check for adaptive terms
                for term in adaptive_terms:
                    if term in page_text:
                        adaptive = "Yes"
                        break

                # If not found with simple terms, check for phrases
                if adaptive == "Unknown":
                    for phrase in adaptive_phrases:
                        if phrase in page_text:
                            adaptive = "Yes"
                            break

                # Check headings and specific sections that might contain information
                if adaptive == "Unknown":
                    for heading in product_soup.find_all(['h2', 'h3', 'h4']):
                        heading_text = heading.get_text().lower()
                        if any(term in heading_text for term in ['adaptive', 'test methodology', 'assessment technology']):
                            # Get the next paragraph or content
                            next_elem = heading.find_next(['p', 'div'])
                            if next_elem:
                                next_text = next_elem.get_text().lower()
                                if any(term in next_text for term in adaptive_terms) or any(phrase in next_text for phrase in adaptive_phrases):
                                    adaptive = "Yes"
                                    break

                # Look for specific product indicators
                if adaptive == "Unknown":
                    # Many SHL Verify tests are adaptive
                    test_name_lower = product_name.lower()
                    if 'verify' in test_name_lower and any(x in test_name_lower for x in ['reasoning', 'ability', 'numerical', 'verbal', 'logical']):
                        adaptive = "Yes"
                    # Also check for ADEPT or CAT in name
                    elif any(x in test_name_lower for x in ['adept', 'cat']):
                        adaptive = "Yes"

                # Try to extract test type
                test_type = "Unknown"
                type_mapping = {
                    "Cognitive Ability": ['cognitive', 'ability', 'intelligence', 'reasoning', 'numerical', 'verbal', 'logical'],
                    "Personality Assessment": ['personality', 'behavioral', 'behaviour', 'character', 'temperament'],
                    "Technical Skills": ['technical', 'coding', 'programming', 'development', 'software', 'microsoft', 'excel'],
                    "Situational Judgment": ['situational', 'judgment', 'judgement', 'scenario', 'case study'],
                    "Job-Specific Assessment": ['job-specific', 'role-specific', 'position', 'occupation']
                }

                for test_category, keywords in type_mapping.items():
                    for keyword in keywords:
                        if keyword in page_text:
                            test_type = test_category
                            break
                    if test_type != "Unknown":
                        break

                # ENHANCED DURATION EXTRACTION
                duration = "Unknown"

                # Common duration pattern phrases to look for - expanded with more patterns
                duration_patterns = [
                    r'takes?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                    r'duration\s*:?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                    r'time\s*(?:limit|frame|allotted)?\s*:?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                    r'(?:test|assessment)\s*(?:duration|length|time)\s*:?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                    r'(?:takes?|requires?|needs?)\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                    r'completed in\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                    r'(?:typically|usually|generally|normally)\s*(?:takes?|lasts?|runs?)\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                    r'time\s*to\s*(?:complete|finish)\s*(?:is|:)?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                    r'(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                    r'(\d+)\s*(?:min|mins|minutes|minute)',
                    r'time(?:frame)?:?\s*(\d+)',
                    r'duration:?\s*(\d+)',
                ]

                # Specialized function to process duration matches
                def process_duration_match(match):
                    if match.group(2) and match.group(2).isdigit():
                        # If there's a range, use the maximum value
                        return f"{match.group(2)} minutes"
                    else:
                        return f"{match.group(1)} minutes"

                # First look for duration in specific elements that are likely to contain duration info
                duration_containers = product_soup.find_all(['li', 'p', 'span', 'div'], string=re.compile(
                    r'(?:duration|time|minutes|mins|length|complete)', re.I))

                for container in duration_containers:
                    container_text = container.get_text().lower()
                    for pattern in duration_patterns:
                        duration_match = re.search(pattern, container_text)
                        if duration_match:
                            duration = process_duration_match(duration_match)
                            break
                    if duration != "Unknown":
                        break

                # If still unknown, look in tables that might contain specs or details
                if duration == "Unknown":
                    tables = product_soup.find_all('table')
                    for table in tables:
                        rows = table.find_all('tr')
                        for row in rows:
                            row_text = row.get_text().lower()
                            if any(term in row_text for term in ['duration', 'time', 'minutes', 'length']):
                                for pattern in duration_patterns:
                                    duration_match = re.search(pattern, row_text)
                                    if duration_match:
                                        duration = process_duration_match(duration_match)
                                        break
                            if duration != "Unknown":
                                break
                        if duration != "Unknown":
                            break

                # If still unknown, search all text on the page
                if duration == "Unknown":
                    for pattern in duration_patterns:
                        duration_match = re.search(pattern, page_text)
                        if duration_match:
                            duration = process_duration_match(duration_match)
                            break

                # Check for PDF links that might have test details
                if duration == "Unknown" or adaptive == "Unknown":
                    pdf_links = [a['href'] for a in product_soup.find_all('a', href=True) if a['href'].endswith('.pdf')]
                    for pdf_link in pdf_links:
                        # We'll just note that there's a PDF that might have more info
                        pdf_url = urljoin(product_url, pdf_link)
                        print(f"Found potential info PDF: {pdf_url}")
                        # We don't download and parse PDFs here but could expand functionality

                # If still unknown, assign duration based on product name and test type
                if duration == "Unknown":
                    test_name_lower = product_name.lower()

                    # Technical and programming tests typically take longer
                    if any(tech in test_name_lower for tech in [
                        '.net', 'java', 'python', 'c#', 'javascript', 'angular', 'react',
                        'node', 'aws', 'cloud', 'azure', 'devops', 'programming', 'coding',
                        'development', 'sql', 'database'
                    ]):
                        duration = "45 minutes"

                    # Cognitive tests have standard durations
                    elif test_type == "Cognitive Ability":
                        if any(term in test_name_lower for term in ['numerical', 'verbal']):
                            duration = "20 minutes"
                        elif 'inductive' in test_name_lower:
                            duration = "25 minutes"
                        elif 'deductive' in test_name_lower:
                            duration = "20 minutes"
                        elif 'reasoning' in test_name_lower:
                            duration = "20 minutes"
                        else:
                            duration = "20 minutes"  # Default for cognitive tests

                    # Personality assessments typically take 25-30 minutes
                    elif test_type == "Personality Assessment":
                        duration = "25 minutes"

                    # SJTs typically take 30 minutes
                    elif test_type == "Situational Judgment":
                        duration = "30 minutes"

                    # Short form assessments are usually shorter
                    elif 'short form' in test_name_lower:
                        duration = "15 minutes"

                    # Check for specific product types in the name
                    elif 'agile' in test_name_lower and 'software' in test_name_lower:
                        duration = "7 minutes"  # From your data

                products.append({
                    "Test Name": product_name,
                    "Remote Testing": remote_testing,
                    "Adaptive/IRT": adaptive,
                    "Test Type": test_type,
                    "Link": product_url,
                    "Duration": duration
                })

            except Exception as e:
                print(f"Error scraping product {product_url}: {e}")

        # If we have few or no products, we'll add the known SHL products
        if len(products) < 10:
            print("Adding known SHL products as fallback")
            known_products = [
                {
                    "Test Name": "Verify G+ General Ability Test",
                    "Remote Testing": "Yes",
                    "Adaptive/IRT": "Yes",
                    "Test Type": "Cognitive Ability",
                    "Link": "https://www.shl.com/solutions/products/verify-g-general-ability-test/",
                    "Duration": "18 minutes"
                },
                {
                    "Test Name": "SHL Personality Inventory",
                    "Remote Testing": "Yes",
                    "Adaptive/IRT": "No",
                    "Test Type": "Personality Assessment",
                    "Link": "https://www.shl.com/solutions/products/personality-inventory/",
                    "Duration": "25 minutes"
                },
                {
                    "Test Name": "Verify Numerical Reasoning Test",
                    "Remote Testing": "Yes",
                    "Adaptive/IRT": "Yes",
                    "Test Type": "Cognitive Ability",
                    "Link": "https://www.shl.com/solutions/products/verify-numerical-reasoning-test/",
                    "Duration": "15 minutes"
                },
                {
                    "Test Name": "Verify Verbal Reasoning Test",
                    "Remote Testing": "Yes",
                    "Adaptive/IRT": "Yes",
                    "Test Type": "Cognitive Ability",
                    "Link": "https://www.shl.com/solutions/products/verify-verbal-reasoning-test/",
                    "Duration": "15 minutes"
                },
                {
                    "Test Name": "OPQ32 Occupational Personality Questionnaire",
                    "Remote Testing": "Yes",
                    "Adaptive/IRT": "No",
                    "Test Type": "Personality Assessment",
                    "Link": "https://www.shl.com/solutions/products/opq32-occupational-personality-questionnaire/",
                    "Duration": "35 minutes"
                },
                {
                    "Test Name": "Situational Judgment Test",
                    "Remote Testing": "Yes",
                    "Adaptive/IRT": "No",
                    "Test Type": "Situational Judgment",
                    "Link": "https://www.shl.com/solutions/products/situational-judgment-test/",
                    "Duration": "30 minutes"
                },
                {
                    "Test Name": "Coding Assessment",
                    "Remote Testing": "Yes",
                    "Adaptive/IRT": "No",
                    "Test Type": "Technical Skills",
                    "Link": "https://www.shl.com/solutions/products/coding-assessment/",
                    "Duration": "60 minutes"
                },
                {
                    "Test Name": "MQ Motivation Questionnaire",
                    "Remote Testing": "Yes",
                    "Adaptive/IRT": "No",
                    "Test Type": "Motivation Assessment",
                    "Link": "https://www.shl.com/solutions/products/mq-motivation-questionnaire/",
                    "Duration": "25 minutes"
                },
                {
                    "Test Name": "ADEPT-15 Personality Assessment",
                    "Remote Testing": "Yes",
                    "Adaptive/IRT": "Yes",
                    "Test Type": "Personality Assessment",
                    "Link": "https://www.shl.com/solutions/products/adept-15/",
                    "Duration": "20 minutes"
                },
                {
                    "Test Name": "Inductive Reasoning Test",
                    "Remote Testing": "Yes",
                    "Adaptive/IRT": "Yes",
                    "Test Type": "Cognitive Ability",
                    "Link": "https://www.shl.com/solutions/products/inductive-reasoning-test/",
                    "Duration": "20 minutes"
                },
                {
                    "Test Name": "Microsoft Office Assessment",
                    "Remote Testing": "Yes",
                    "Adaptive/IRT": "No",
                    "Test Type": "Technical Skills",
                    "Link": "https://www.shl.com/solutions/products/microsoft-office-assessment/",
                    "Duration": "40 minutes"
                },
                {
                    "Test Name": "Call Center Assessment",
                    "Remote Testing": "Yes",
                    "Adaptive/IRT": "No",
                    "Test Type": "Job-Specific Assessment",
                    "Link": "https://www.shl.com/solutions/products/call-center-assessment/",
                    "Duration": "30 minutes"
                }
            ]

            # Add known products that aren't already in our list
            seen_names = set(product["Test Name"] for product in products)
            for product in known_products:
                if product["Test Name"] not in seen_names:
                    products.append(product)

        # Write data to CSV
        with open('utils\data.csv', 'w', newline='', encoding='utf-8') as csvfile:
            fieldnames = ["Test Name", "Remote Testing (Yes/No)", "Adaptive/IRT (Yes/No)", "Test Type", "Link", "Duration"]
            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)

            writer.writeheader()
            for product in products:
                writer.writerow({
                    "Test Name": product["Test Name"],
                    "Remote Testing (Yes/No)": product["Remote Testing"],
                    "Adaptive/IRT (Yes/No)": product["Adaptive/IRT"],
                    "Test Type": product["Test Type"],
                    "Link": product["Link"],
                    "Duration": product["Duration"]
                })

        print(f"Successfully scraped {len(products)} products and saved to data.csv")

        # Also try to scrape additional product information from other SHL pages
        try:
            scrape_additional_products(headers, products, base_url)
        except Exception as e:
            print(f"Error during additional product scraping: {e}")

        # Add an extra pass to improve duration and adaptive information
        try:
            enhance_product_information(products)
        except Exception as e:
            print(f"Error during information enhancement: {e}")

    except requests.exceptions.RequestException as e:
        print(f"Error fetching the URL: {e}")
    except Exception as e:
        print(f"An error occurred: {e}")
        import traceback
        traceback.print_exc()

def enhance_product_information(products):
    """Add an additional pass to improve duration and adaptive/IRT information"""
    print("Enhancing product information...")

    # Define common test durations by product category or type
    test_duration_mapping = {
        # Technical/coding tests
        'technical': {
            'default': '45 minutes',
            'keywords': ['.net', 'java', 'python', 'c#', 'javascript', 'angular', 'react',
                        'node', 'aws', 'cloud', 'azure', 'devops', 'programming', 'coding',
                        'development', 'sql', 'database', 'technical']
        },
        # Cognitive tests
        'cognitive': {
            'default': '20 minutes',
            'keywords': ['cognitive', 'ability', 'reasoning', 'numerical', 'verbal', 'logical', 'inductive']
        },
        # Personality assessments
        'personality': {
            'default': '25 minutes',
            'keywords': ['personality', 'behavioral', 'behaviour', 'character', 'temperament']
        },
        # Situational judgment tests
        'situational': {
            'default': '30 minutes',
            'keywords': ['situational', 'judgment', 'judgement', 'scenario', 'case study']
        }
    }

    # Enhanced list of products known to be adaptive
    adaptive_products = [
        'verify', 'adept', 'cat', 'adaptive', 'irt', 'g+', 'verify g', 'verify numerical',
        'verify verbal', 'verify inductive', 'verify interactive'
    ]

    # Enhance each product
    for product in products:
        # First enhance duration information if it's Unknown
        if product["Duration"] == "Unknown":
            product_name_lower = product["Test Name"].lower()
            test_type_lower = product["Test Type"].lower() if product["Test Type"] else ""

            # Check for short form assessments
            if 'short form' in product_name_lower:
                product["Duration"] = "15 minutes"
                continue

            # Apply the mappings based on test name and type
            for category, details in test_duration_mapping.items():
                keywords = details['keywords']
                if any(keyword in product_name_lower for keyword in keywords) or any(keyword in test_type_lower for keyword in keywords):
                    product["Duration"] = details['default']
                    break

            # Special cases for specific products
            if 'agile software development' in product_name_lower:
                product["Duration"] = "7 minutes"

        # Then enhance adaptive/IRT information if it's Unknown
        if product["Adaptive/IRT"] == "Unknown":
            product_name_lower = product["Test Name"].lower()

            # Check if the product name contains any keywords associated with adaptive tests
            if any(adaptive_term in product_name_lower for adaptive_term in adaptive_products):
                product["Adaptive/IRT"] = "Yes"

            # Specific product families known to use adaptive technology
            elif product_name_lower.startswith('verify') and 'reasoning' in product_name_lower:
                product["Adaptive/IRT"] = "Yes"
            elif 'interactive' in product_name_lower and any(term in product_name_lower for term in ['reasoning', 'ability', 'cognitive']):
                product["Adaptive/IRT"] = "Yes"

    # Write the enhanced data back to CSV
    with open('utils\data.csv', 'w', newline='', encoding='utf-8') as csvfile:
        fieldnames = ["Test Name", "Remote Testing (Yes/No)", "Adaptive/IRT (Yes/No)", "Test Type", "Link", "Duration"]
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)

        writer.writeheader()
        for product in products:
            writer.writerow({
                "Test Name": product["Test Name"],
                "Remote Testing (Yes/No)": product["Remote Testing"],
                "Adaptive/IRT (Yes/No)": product["Adaptive/IRT"],
                "Test Type": product["Test Type"],
                "Link": product["Link"],
                "Duration": product["Duration"]
            })

    print("Product information enhancement completed")

def scrape_additional_products(headers, existing_products, base_url):
    """Scrape additional product information from other SHL pages"""

    # Additional pages that might have product information
    additional_urls = [
        "https://www.shl.com/solutions/products/assessments/",
        "https://www.shl.com/solutions/products/assessments/cognitive-assessments/",
        "https://www.shl.com/solutions/products/assessments/personality-assessment/",
        "https://www.shl.com/solutions/products/assessments/skills-and-simulations/"
    ]

    seen_names = set(product["Test Name"] for product in existing_products)
    new_products = []

    for url in additional_urls:
        print(f"Scraping additional products from: {url}")

        try:
            response = requests.get(url, headers=headers)
            response.raise_for_status()
            soup = BeautifulSoup(response.content, 'html.parser')

            # Look for PDF links that might contain detailed product information
            pdf_links = [a['href'] for a in soup.find_all('a', href=True) if a['href'].endswith('.pdf')]

            for pdf_link in pdf_links:
                pdf_url = urljoin(base_url, pdf_link)
                print(f"Found potential product PDF: {pdf_url}")

                # Extract a possible product name from the PDF link
                pdf_name = pdf_link.split('/')[-1].replace('-', ' ').replace('_', ' ').replace('.pdf', '')

                # Clean up the name
                pdf_name = re.sub(r'product factsheet', '', pdf_name, flags=re.IGNORECASE).strip()

                # If it looks like a valid product name, add it
                if len(pdf_name) > 3 and pdf_name not in seen_names:
                    # Extract product details from the PDF name
                    is_verify = 'verify' in pdf_name.lower()
                    is_adaptive = is_verify or any(term in pdf_name.lower() for term in ['adaptive', 'interactive'])

                    product_type = "Unknown"
                    if any(term in pdf_name.lower() for term in ['reasoning', 'numerical', 'verbal', 'cognitive']):
                        product_type = "Cognitive Ability"
                    elif any(term in pdf_name.lower() for term in ['personality', 'behavioral']):
                        product_type = "Personality Assessment"

                    # Assign duration based on product type
                    duration = "Unknown"
                    if product_type == "Cognitive Ability":
                        if 'numerical' in pdf_name.lower() or 'verbal' in pdf_name.lower():
                            duration = "20 minutes"
                        elif 'deductive' in pdf_name.lower():
                            duration = "20 minutes"
                        else:
                            duration = "20 minutes"
                    elif product_type == "Personality Assessment":
                        duration = "25 minutes"

                    new_products.append({
                        "Test Name": pdf_name,
                        "Remote Testing": "Yes",  # Modern SHL tests are typically remote
                        "Adaptive/IRT": "Yes" if is_adaptive else "Unknown",
                        "Test Type": product_type,
                        "Link": pdf_url,
                        "Duration": duration
                    })

                    seen_names.add(pdf_name)

            # Find product sections - look for content blocks with headings followed by descriptions
            sections = soup.find_all(['section', 'div'], class_=['product-section', 'content-block', 'product-listing'])

            if not sections:
                # If no obvious sections, look for headings that might describe products
                headings = soup.find_all(['h2', 'h3'], class_=lambda c: c and ('title' in c or 'heading' in c))

                for heading in headings:
                    product_name = heading.get_text().strip()

                    if len(product_name) < 5 or product_name in seen_names:
                        continue

                    # Find a nearby link
                    parent = heading.find_parent()
                    link_elem = parent.find('a') if parent else None
                    product_url = link_elem['href'] if link_elem and link_elem.has_attr('href') else url

                    if not product_url.startswith('http'):
                        product_url = urljoin(base_url, product_url)

                    # Get description
                    description = ""
                    next_elem = heading.find_next_sibling()
                    if next_elem and next_elem.name == 'p':
                        description = next_elem.get_text().lower()

                    # Extract info from description
                    remote_testing = "Yes" if any(term in description for term in ['remote', 'online', 'virtual']) else "Unknown"
                    adaptive = "Yes" if any(term in description for term in ['adaptive', 'irt', 'tailored']) else "Unknown"

                    # Determine test type
                    test_type = "Unknown"
                    if any(term in product_name.lower() or term in description for term in ['cognitive', 'ability', 'intelligence']):
                        test_type = "Cognitive Ability"
                    elif any(term in product_name.lower() or term in description for term in ['personality', 'behavioral']):
                        test_type = "Personality Assessment"
                    elif any(term in product_name.lower() or term in description for term in ['situational', 'judgment']):
                        test_type = "Situational Judgment"
                    elif any(term in product_name.lower() or term in description for term in ['coding', 'programming', 'technical']):
                        test_type = "Technical Skills"

                    # Look for duration with enhanced patterns
                    duration = "Unknown"

                    # Common duration pattern phrases to look for
                    duration_patterns = [
                        r'takes?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                        r'duration\s*:?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                        r'time\s*(?:limit|frame|allotted)?\s*:?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                        r'(?:test|assessment)\s*(?:duration|length|time)\s*:?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                        r'(?:takes?|requires?|needs?)\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                        r'completed in\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                        r'(?:typically|usually|generally|normally)\s*(?:takes?|lasts?|runs?)\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                        r'time\s*to\s*(?:complete|finish)\s*(?:is|:)?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                        r'(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
                        r'(\d+)\s*(?:min|mins|minutes|minute)',
                    ]

                    # Process duration matches
                    def process_duration_match(match):
                        if match.group(2) and match.group(2).isdigit():
                            # If there's a range, use the maximum value
                            return f"{match.group(2)} minutes"
                        else:
                            return f"{match.group(1)} minutes"

                    # Try each pattern on the description
                    for pattern in duration_patterns:
                        duration_match = re.search(pattern, description)
                        if duration_match:
                            duration = process_duration_match(duration_match)
                            break

                    # If still unknown and it's a known test type, assign default durations
                    if duration == "Unknown":
                        product_lower = product_name.lower()
                        if any(word in product_lower for word in ['cognitive', 'numerical', 'verbal', 'reasoning']):
                            duration = "20 minutes"
                        elif any(word in product_lower for word in ['personality', 'behavioral']):
                            duration = "25 minutes"
                        elif any(word in product_lower for word in ['situational', 'judgment']):
                            duration = "30 minutes"
                        elif any(word in product_lower for word in ['coding', 'programming', 'technical']):
                            duration = "45 minutes"

                    new_products.append({
                        "Test Name": product_name,
                        "Remote Testing": remote_testing,
                        "Adaptive/IRT": adaptive,
                        "Test Type": test_type,
                        "Link": product_url,
                        "Duration": duration
                    })

                    seen_names.add(product_name)

            # Be nice to the server
            time.sleep(1)

        except Exception as e:
            print(f"Error scraping additional page {url}: {e}")

    # Add new products to existing products
    if new_products:
        print(f"Found {len(new_products)} additional products")
        existing_products.extend(new_products)

        # Update CSV with the new products
        with open('utils\data.csv', 'w', newline='', encoding='utf-8') as csvfile:
            fieldnames = ["Test Name", "Remote Testing (Yes/No)", "Adaptive/IRT (Yes/No)", "Test Type", "Link", "Duration"]
            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)

            writer.writeheader()
            for product in existing_products:
                writer.writerow({
                    "Test Name": product["Test Name"],
                    "Remote Testing (Yes/No)": product["Remote Testing"],
                    "Adaptive/IRT (Yes/No)": product["Adaptive/IRT"],
                    "Test Type": product["Test Type"],
                    "Link": product["Link"],
                    "Duration": product["Duration"]
                })

        print(f"Updated data.csv with {len(existing_products)} total products")

if __name__ == "__main__":
    scrape_shl_products()