# Web scraping for Hacker News
from bs4 import BeautifulSoup
import requests

def scrape_hn_thread(thread_id):
    # Fetch and parse HTML
    # Extract job postings from comments
    # Structure: company, role, location, description
    pass

# API integration for Adzuna
import requests

def fetch_adzuna_jobs(params):
    endpoint = "https://api.adzuna.com/v1/api/jobs"
    # Query with tech-related keywords
    # Paginate through results
    # Checkpoint saves for reliability
    pass

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from IPython.display import Image, display
import warnings
warnings.filterwarnings('ignore')

plt.style.use('seaborn-v0_8-darkgrid')
pd.set_option('display.max_columns', None)

monthly_summary = pd.read_csv('data/processed/monthly_time_series_summary.csv', index_col=0)
monthly_summary.index = pd.to_datetime(monthly_summary.index)

print(" Analysis results loaded")
print(f"   Months of data: {len(monthly_summary)}")
print(f"\n Summary Statistics:")
print(monthly_summary[['AI/ML', 'General IT', 'Hybrid', 'Total']].describe())

total_ai = monthly_summary['AI/ML'].sum()
total_it = monthly_summary['General IT'].sum()
total_hybrid = monthly_summary['Hybrid'].sum()
total_tech = total_ai + total_it + total_hybrid

print(" MARKET SHARE ANALYSIS")
print("="*60)
print(f"\nTotal Technology Job Postings: {total_tech:,}")
print(f"\nBreakdown:")
print(f"  • AI/ML:       {total_ai:>5,} jobs ({(total_ai/total_tech)*100:>5.1f}%)")
print(f"  • General IT:  {total_it:>5,} jobs ({(total_it/total_tech)*100:>5.1f}%)")
print(f"  • Hybrid:      {total_hybrid:>5,} jobs ({(total_hybrid/total_tech)*100:>5.1f}%)")

print(f"\nKey Metric:")
print(f"  AI:IT Ratio = {total_ai/total_it:.3f}:1")
print(f"  (For every {int(total_it/total_ai)} General IT jobs, there is 1 AI/ML job)")

display(Image(filename='data/processed/executive_dashboard.png'))

print(" TREND ANALYSIS\n" + "="*60)

display(Image(filename='data/processed/time_series_trends.png'))

print("\nObservations:")
print("  • Both AI/ML and General IT show temporal variation")
print("  • Hybrid roles represent emerging category")
print("  • Clear visualization of market composition over time")

print(" COMPANY ANALYSIS\n" + "="*60)

display(Image(filename='data/processed/top_ai_companies.png'))

print("\nTop AI/ML Hiring Companies:")
print("  • Identifies market leaders in AI talent acquisition")
print("  • Shows concentration of AI hiring among specific companies")
print("  • Useful for understanding industry AI adoption patterns")

display(Image(filename='data/processed/company_positioning_scatter.png'))

print("\nKey Insights:")
print("  • Bubble size represents total hiring volume")
print("  • Y-axis shows % of AI roles (higher = more AI-focused)")
print("  • Companies above 50% line are AI-heavy")
print("  • Identifies strategic positioning: AI-first vs traditional IT")

from scipy import stats

print(" STATISTICAL ANALYSIS\n" + "="*60)

monthly_summary['month_numeric'] = range(len(monthly_summary))

ai_slope, ai_intercept, ai_r, ai_p, ai_stderr = stats.linregress(
    monthly_summary['month_numeric'], 
    monthly_summary['AI/ML']
)

it_slope, it_intercept, it_r, it_p, it_stderr = stats.linregress(
    monthly_summary['month_numeric'],
    monthly_summary['General IT']
)

print("\n1. Linear Regression Results:")
print(f"\n   AI/ML Trend:")
print(f"      Slope: {ai_slope:+.2f} jobs/month")
print(f"      R²: {ai_r**2:.4f}")
print(f"      P-value: {ai_p:.4f}")
print(f"      Significance: {' Significant' if ai_p < 0.05 else ' Not significant'} (α=0.05)")

print(f"\n   General IT Trend:")
print(f"      Slope: {it_slope:+.2f} jobs/month")
print(f"      R²: {it_r**2:.4f}")
print(f"      P-value: {it_p:.4f}")
print(f"      Significance: {' Significant' if it_p < 0.05 else ' Not significant'} (α=0.05)")

print(f"\n2. Comparative Growth:")
if ai_slope > it_slope:
    print(f"    AI/ML growing FASTER than General IT")
    print(f"    Difference: {ai_slope - it_slope:+.2f} jobs/month")
else:
    print(f"    General IT growing faster than AI/ML")
    print(f"    Difference: {it_slope - ai_slope:+.2f} jobs/month")

print(f"\n3. Average Month-over-Month Growth:")
avg_ai_mom = monthly_summary['AI_ML_MoM_%'].mean()
avg_it_mom = monthly_summary['General_IT_MoM_%'].mean()
print(f"   AI/ML: {avg_ai_mom:.1f}% per month")
print(f"   General IT: {avg_it_mom:.1f}% per month")

print(" HYPOTHESIS TESTING RESULTS\n" + "="*60)

print("\n**Primary Hypothesis:**")
print("'AI/ML roles growing faster than General IT roles'")
if ai_slope > it_slope:
    print(" SUPPORTED by data")
    print(f"   Evidence: AI slope ({ai_slope:+.2f}) > IT slope ({it_slope:+.2f})")
else:
    print(" NOT SUPPORTED by data")
    print(f"   Evidence: IT slope ({it_slope:+.2f}) > AI slope ({ai_slope:+.2f})")

print("\n**Sub-Hypothesis 1:**")
print("'AI/ML shows positive growth trend'")
if ai_slope > 0:
    print(f" SUPPORTED (slope = {ai_slope:+.2f})")
else:
    print(f" NOT SUPPORTED (slope = {ai_slope:+.2f})")

print("\n**Sub-Hypothesis 2:**")
print("'AI:IT ratio increasing over time'")
ratio_first = monthly_summary['AI_IT_ratio'].iloc[0]
ratio_last = monthly_summary['AI_IT_ratio'].iloc[-1]
ratio_change = ratio_last - ratio_first
if ratio_change > 0:
    print(f" SUPPORTED ({ratio_first:.3f}  {ratio_last:.3f}, +{ratio_change:.3f})")
else:
    print(f" NOT SUPPORTED ({ratio_first:.3f}  {ratio_last:.3f}, {ratio_change:.3f})")

print("\n**Sub-Hypothesis 4:**")
print("'Trends are statistically significant'")
alpha = 0.05
if ai_p < alpha or it_p < alpha:
    print(f" PARTIALLY SUPPORTED")
    print(f"   AI p-value: {ai_p:.4f} ({'significant' if ai_p < alpha else 'not significant'})")
    print(f"   IT p-value: {it_p:.4f} ({'significant' if it_p < alpha else 'not significant'})")
else:
    print(f" NOT SUPPORTED (need more data)")
    print(f"   Both p-values > {alpha}")

pip install -r requirements.txt

Job Market Analytics: AI/ML vs General IT Trends - Final Report¶

Executive Summary¶

Key Findings:¶

1. Introduction & Hypothesis¶

1.1 Background¶

1.2 Research Hypothesis¶

1.3 Research Questions¶

2. Methodology¶

2.1 Data Collection¶

2.2 Data Cleaning Pipeline¶

2.3 Job Categorization¶

2.4 Time Series Analysis¶

3. Key Findings¶

3.1 Load Analysis Results¶

3.2 Market Composition¶

3.3 Executive Dashboard¶

3.4 Detailed Trend Analysis¶

4. Statistical Validation¶

4.1 Growth Rate Analysis¶

4.2 Hypothesis Testing Results¶

5. Limitations & Considerations¶

5.1 Data Collection Limitations¶

5.2 Categorization Limitations¶

5.3 Statistical Limitations¶

5.4 External Validity¶

5.5 Recommendations for Future Research¶

6. Conclusions¶

6.1 Summary of Findings¶

6.2 Implications¶

6.3 Future Work¶

6.4 Final Thoughts¶

6.5 Acknowledgments¶

Appendix: Reproducibility¶

A. Repository Structure¶

B. Execution Instructions¶

C. Key Files¶