# projects/article-quality-assessment/model.py

import requests
import json
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.ensemble import RandomForestClassifier
import numpy as np

class ArticleQualityModel:
    """
    Machine Learning model to assess Wikipedia article quality.
    Uses features like length, references, images, and structure.
    """
    
    def __init__(self):
        self.vectorizer = TfidfVectorizer(max_features=500)
        self.classifier = RandomForestClassifier(n_estimators=100)
        self.is_trained = False
    
    def extract_features(self, article_text):
        """
        Extract structural features from article text.
        
        Args:
            article_text (str): Raw Wikipedia article text
            
        Returns:
            dict: Feature dictionary
        """
        features = {
            'length': len(article_text),
            'word_count': len(article_text.split()),
            'section_count': article_text.count('=='),
            'reference_count': article_text.count('[[') + article_text.count(']]'),
            'image_count': article_text.count('[[File:'),
            'internal_links': article_text.count('[[Wikipedia:'),
            'sentence_count': len(article_text.split('.'))
        }
        return features
    
    def train(self, X_text, y_labels):
        """
        Train the model on labeled article data.
        
        Args:
            X_text (list): List of article texts
            y_labels (list): Quality labels (0=stub, 1=start, 2=c-class, 3=b-class)
        """
        X_features = self._transform_features(X_text)
        self.classifier.fit(X_features, y_labels)
        self.is_trained = True
    
    def predict(self, article_text):
        """
        Predict article quality.
        
        Args:
            article_text (str): Wikipedia article text
            
        Returns:
            dict: Quality prediction with confidence scores
        """
        if not self.is_trained:
            raise ValueError("Model has not been trained yet.")
        
        features = self._transform_features([article_text])
        prediction = self.classifier.predict(features)
        probabilities = self.classifier.predict_proba(features)
        
        quality_labels = ['Stub', 'Start', 'C-Class', 'B-Class']
        
        return {
            'quality': quality_labels[prediction[0]],
            'confidence': np.max(probabilities[0]) * 100,
            'all_scores': {
                label: prob * 100 
                for label, prob in zip(quality_labels, probabilities[0])
            }
        }
    
    def _transform_features(self, texts):
        """Transform texts into feature vectors."""
        return self.vectorizer.transform(texts)
    
    def save(self, path='model.pkl'):
        """Save the trained model to disk."""
        import joblib
        joblib.dump({
            'classifier': self.classifier,
            'vectorizer': self.vectorizer
        }, path)
    
    def load(self, path='model.pkl'):
        """Load a trained model from disk."""
        import joblib
        data = joblib.load(path)
        self.classifier = data['classifier']
        self.vectorizer = data['vectorizer']
        self.is_trained = True

# projects/commons-bot/bot.py

import pywikibot
from pywikibot import pagegenerators
import time
import os

class CommonsUploadBot:
    """
    Automated bot for uploading and categorizing files on Wikimedia Commons.
    """
    
    def __init__(self, username=None, password=None):
        """
        Initialize the bot with authentication.
        
        Args:
            username (str): Wikimedia Commons username
            password (str): Wikimedia Commons password
        """
        self.site = pywikibot.Site('commons', 'commons')
        
        if username and password:
            # Login using credentials
            self.site.login(username, password)
        
        self.uploaded_files = []
        self.failed_uploads = []
    
    def upload_file(self, file_path, title, description, categories, author=None):
        """
        Upload a single file to Wikimedia Commons.
        
        Args:
            file_path (str): Local path to the file
            title (str): Desired title on Commons
            description (str): File description
            categories (list): List of categories to add
            author (str): File author (defaults to username)
            
        Returns:
            bool: True if successful, False otherwise
        """
        try:
            if not os.path.exists(file_path):
                raise FileNotFoundError(f"File not found: {file_path}")
            
            # Prepare upload parameters using pywikibot's upload method
            page = pywikibot.Page(self.site, f"File:{title}")
            
            # Create the file page content
            content = f"""== {{int:filedesc}} ==
{{Information
|description={description}
|date={time.strftime('%Y-%m-%d')}
|source={{own}}
|author={author or self.site.user()}
|permission=
|other versions=
}}

== {{int:license-header}} ==
{{self|cc-by-sa-4.0}}

"""
            # Add categories
            for category in categories:
                content += f"\n[[Category:{category}]]"
            
            # Upload the file
            page.text = content
            page.upload(file_path, filename=title, description=description)
            
            self.uploaded_files.append(title)
            print(f"✅ Successfully uploaded: {title}")
            return True
                
        except Exception as e:
            self.failed_uploads.append(title)
            print(f"❌ Error uploading {title}: {str(e)}")
            return False
    
    def batch_upload(self, file_list):
        """
        Upload multiple files in batch.
        
        Args:
            file_list (list): List of (file_path, title, description, categories) tuples
            
        Returns:
            dict: Upload summary
        """
        for file_path, title, description, categories in file_list:
            self.upload_file(file_path, title, description, categories)
            # Rate limiting to avoid API throttling
            time.sleep(2)
        
        return {
            'successful': len(self.uploaded_files),
            'failed': len(self.failed_uploads),
            'uploaded_files': self.uploaded_files,
            'failed_files': self.failed_uploads
        }
    
    def add_category(self, file_title, category):
        """
        Add a category to an already uploaded file.
        
        Args:
            file_title (str): Title of the file on Commons
            category (str): Category to add
            
        Returns:
            bool: True if successful, False otherwise
        """
        try:
            page = pywikibot.Page(self.site, f"File:{file_title}")
            text = page.text
            new_category = f"[[Category:{category}]]"
            
            if new_category not in text:
                text += f"\n{new_category}"
                page.text = text
                page.save("Added category")
                return True
            return True
            
        except Exception as e:
            print(f"❌ Error adding category: {str(e)}")
            return False
    
    def get_upload_report(self):
        """
        Generate a report of all uploads.
        
        Returns:
            dict: Upload statistics
        """
        return {
            'total_uploads': len(self.uploaded_files),
            'failed_uploads': len(self.failed_uploads),
            'success_rate': (
                len(self.uploaded_files) / 
                (len(self.uploaded_files) + len(self.failed_uploads)) * 100
            ) if (len(self.uploaded_files) + len(self.failed_uploads)) > 0 else 0
        }

# projects/wikidata-dashboard/query.py

from SPARQLWrapper import SPARQLWrapper, JSON
import pandas as pd

class WikidataQuery:
    """
    Handles SPARQL queries to Wikidata.
    """
    
    ENDPOINT = "https://query.wikidata.org/sparql"
    
    def __init__(self):
        self.sparql = SPARQLWrapper(self.ENDPOINT)
        self.sparql.setReturnFormat(JSON)
    
    def get_scientist_count_by_year(self, country_code='Q156'):
        """
        Query to get number of scientists by birth year for a country.
        
        Args:
            country_code (str): Wikidata ID of the country
            
        Returns:
            DataFrame: Scientists by year
        """
        query = f"""
        SELECT ?year (COUNT(DISTINCT ?scientist) AS ?count)
        WHERE {{
            ?scientist wdt:P106 wd:Q901;  # occupation: scientist
                       wdt:P19 ?birthplace. # place of birth
            ?birthplace wdt:P17 wd:{country_code}. # country
            ?scientist wdt:P569 ?birthdate.
            BIND(YEAR(?birthdate) AS ?year)
        }}
        GROUP BY ?year
        ORDER BY ?year
        """
        
        self.sparql.setQuery(query)
        results = self.sparql.query().convert()
        
        data = []
        for item in results['results']['bindings']:
            data.append({
                'year': int(item['year']['value']),
                'count': int(item['count']['value'])
            })
        
        return pd.DataFrame(data)
    
    def get_organization_network(self, organization_id):
        """
        Get network of organizations linked to a given organization.
        
        Args:
            organization_id (str): Wikidata ID of the organization
            
        Returns:
            DataFrame: Network links
        """
        query = f"""
        SELECT ?org ?orgLabel ?relationship
        WHERE {{
            wd:{organization_id} ?rel ?org.
            ?org rdfs:label ?orgLabel.
            FILTER(LANG(?orgLabel) = "en")
            VALUES ?rel {{
                wdt:P527  # has part
                wdt:P361  # part of
                wdt:P159  # headquarters location
                wdt:P276  # location
            }}
            BIND(STR(?rel) AS ?relationship)
        }}
        LIMIT 50
        """
        
        self.sparql.setQuery(query)
        results = self.sparql.query().convert()
        
        data = []
        for item in results['results']['bindings']:
            data.append({
                'organization': item.get('orgLabel', {}).get('value', 'Unknown'),
                'relationship': item.get('relationship', 'Unknown')
            })
        
        return pd.DataFrame(data)
    
    def get_wikidata_entity_info(self, entity_id):
        """
        Get basic information about a Wikidata entity.
        
        Args:
            entity_id (str): Wikidata ID (e.g., 'Q42')
            
        Returns:
            dict: Entity information
        """
        query = f"""
        SELECT ?label ?description ?aliases
        WHERE {{
            wd:{entity_id} rdfs:label ?label.
            wd:{entity_id} schema:description ?description.
            OPTIONAL {{
                wd:{entity_id} skos:altLabel ?aliases.
            }}
            FILTER(LANG(?label) = "en")
            FILTER(LANG(?description) = "en")
        }}
        LIMIT 1
        """
        
        self.sparql.setQuery(query)
        results = self.sparql.query().convert()
        
        if results['results']['bindings']:
            return results['results']['bindings'][0]
        return None

# projects/arabic-ocr/pipeline.py

import pytesseract
import cv2
import numpy as np
from PIL import Image
import re

class ArabicOCRPipeline:
    """
    OCR pipeline for Arabic historical documents.
    """
    
    def __init__(self, language='ara'):
        """
        Initialize the OCR pipeline.
        
        Args:
            language (str): Language code ('ara' for Arabic)
        """
        self.language = language
        self.config = f'-l {language} --oem 3 --psm 6'
    
    def preprocess_image(self, image_path):
        """
        Preprocess image for better OCR results.
        
        Args:
            image_path (str): Path to the image file
            
        Returns:
            np.ndarray: Preprocessed image
        """
        # Read image
        image = cv2.imread(image_path)
        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
        
        # Apply noise removal
        denoised = cv2.fastNlMeansDenoising(gray, None, 10, 7, 21)
        
        # Apply thresholding
        _, thresh = cv2.threshold(denoised, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
        
        # Deskew image
        coords = np.column_stack(np.where(thresh > 0))
        if len(coords) > 0:
            angle = cv2.minAreaRect(coords)[-1]
            
            if angle < -45:
                angle = 90 + angle
            if angle != 0:
                (h, w) = thresh.shape[:2]
                center = (w // 2, h // 2)
                M = cv2.getRotationMatrix2D(center, angle, 1.0)
                thresh = cv2.warpAffine(thresh, M, (w, h), 
                                       flags=cv2.INTER_CUBIC, 
                                       borderMode=cv2.BORDER_REPLICATE)
        
        return thresh
    
    def extract_text(self, image_path):
        """
        Extract Arabic text from image.
        
        Args:
            image_path (str): Path to the image file
            
        Returns:
            str: Extracted text
        """
        # Preprocess image
        processed = self.preprocess_image(image_path)
        
        # Convert to PIL Image
        pil_image = Image.fromarray(processed)
        
        # Perform OCR
        text = pytesseract.image_to_string(pil_image, config=self.config)
        
        # Clean text
        text = self._clean_text(text)
        
        return text
    
    def extract_entities(self, text):
        """
        Extract named entities from Arabic text.
        
        Args:
            text (str): Extracted text
            
        Returns:
            dict: Extracted entities
        """
        # Simple regex-based entity extraction
        entities = {
            'dates': re.findall(r'\d{1,2}[/-]\d{1,2}[/-]\d{4}', text),
            'numbers': re.findall(r'\b\d+\b', text),
            'arabic_words': re.findall(r'[أ-ي]{3,}', text)
        }
        
        return entities
    
    def _clean_text(self, text):
        """
        Clean extracted text.
        
        Args:
            text (str): Raw extracted text
            
        Returns:
            str: Cleaned text
        """
        # Remove extra whitespace
        text = re.sub(r'\s+', ' ', text)
        
        # Keep Arabic, numbers, and basic punctuation
        text = re.sub(r'[^أ-ي\s\d\.\,\?\!]', '', text)
        
        return text.strip()
    
    def process_document(self, image_path, output_file=None):
        """
        Process the entire document pipeline.
        
        Args:
            image_path (str): Path to the image file
            output_file (str): Path to save extracted text
            
        Returns:
            dict: Complete pipeline results
        """
        # Extract text
        text = self.extract_text(image_path)
        
        # Extract entities
        entities = self.extract_entities(text)
        
        # Save output
        if output_file:
            with open(output_file, 'w', encoding='utf-8') as f:
                f.write(text)
        
        return {
            'text': text,
            'entities': entities,
            'word_count': len(text.split()),
            'char_count': len(text)
        }

# tests/test_api.py

import pytest
from projects.article-quality-assessment.model import ArticleQualityModel

class TestArticleQualityModel:
    """Test suite for Article Quality Assessment Model."""
    
    @pytest.fixture
    def model(self):
        """Provide a fresh model instance."""
        return ArticleQualityModel()
    
    @pytest.fixture
    def sample_articles(self):
        """Provide sample article texts."""
        return {
            'stub': "This is a short stub article about a topic.",
            'good': """This is a well-developed article with multiple sections.
            
            == Introduction ==
            The topic is important and well-covered.
            
            == History ==
            The history of this topic dates back centuries.
            
            == References ==
            [[1]] Reference one
            [[2]] Reference two
            [[File:image.jpg]] An image illustrating the topic.
            """,
            'featured': """This is a comprehensive featured article.
            
            == Lead Section ==
            The lead provides a complete overview.
            
            == Section 1 ==
            Detailed content about the first aspect.
            
            == Section 2 ==
            Detailed content about the second aspect.
            
            == Section 3 ==
            Detailed content about the third aspect.
            
            == References ==
            Multiple reliable references throughout.
            [[File:image1.jpg]] Important image 1
            [[File:image2.jpg]] Important image 2
            """
        }
    
    def test_extract_features(self, model, sample_articles):
        """Test feature extraction from articles."""
        features = model.extract_features(sample_articles['good'])
        
        assert 'length' in features
        assert 'word_count' in features
        assert 'section_count' in features
        assert 'reference_count' in features
        assert features['length'] > 0
        assert features['word_count'] > 0
    
    def test_train_and_predict(self, model, sample_articles):
        """Test model training and prediction."""
        # Prepare training data
        X_train = list(sample_articles.values())
        y_train = [0, 2, 3]  # stub, good, featured
        
        # Train model
        model.train(X_train, y_train)
        
        # Test prediction
        result = model.predict(sample_articles['stub'])
        
        assert 'quality' in result
        assert 'confidence' in result
        assert 'all_scores' in result
        assert 0 <= result['confidence'] <= 100
    
    def test_save_and_load(self, model, sample_articles, tmp_path):
        """Test saving and loading the model."""
        # Train model
        X_train = list(sample_articles.values())
        y_train = [0, 2, 3]
        model.train(X_train, y_train)
        
        # Save model
        model_path = tmp_path / "test_model.pkl"
        model.save(str(model_path))
        
        # Create new model and load
        new_model = ArticleQualityModel()
        new_model.load(str(model_path))
        
        assert new_model.is_trained
        
        # Test prediction with loaded model
        result = new_model.predict(sample_articles['stub'])
        assert 'quality' in result