HardikSankhla/VoiceInvoiceAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit for fresh VoiceInvoiceAI repo

Browse Source
This commit is contained in:
Hardik Sankhla
2025-06-25 23:03:48 +05:30
commit 6fe387d900
13 changed files with 1113 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

199
backend/services/llm_service.py Normal file
View File

@@ -0,0 +1,199 @@
# backend/services/llm_service.py
import torch
from transformers import Qwen2AudioForConditionalGeneration, AutoProcessor
import librosa
import re
import json
import os
from typing import Dict, Any, Optional
from backend.config import QWEN2_AUDIO_MODEL_NAME, MINIO_AUDIO_BUCKET
from backend.models import InvoiceData, InvoiceItem
from backend.core.utils import clear_gpu_memory, autofill_invoice_data
from backend.services.storage_service import minio_storage_service # Import the MinIO service
# Global variables to hold model instances for efficiency
qwen2_audio_processor = None
qwen2_audio_model = None
device = "cuda" if torch.cuda.is_available() else "cpu"
def load_qwen2_audio_model() -> str:
"""
Loads the Qwen2-Audio model and processor.
Uses 8-bit quantization for GPU to reduce VRAM usage.
"""
global qwen2_audio_processor, qwen2_audio_model, device
if qwen2_audio_model is None or qwen2_audio_processor is None:
try:
print(f"Loading Qwen2-Audio model from {QWEN2_AUDIO_MODEL_NAME} for device: {device}")
qwen2_audio_processor = AutoProcessor.from_pretrained(QWEN2_AUDIO_MODEL_NAME, trust_remote_code=True)
if device == "cuda":
qwen2_audio_model = Qwen2AudioForConditionalGeneration.from_pretrained(
QWEN2_AUDIO_MODEL_NAME,
load_in_8bit=True,
device_map="auto",
torch_dtype=torch.float16,
trust_remote_code=True
)
print("Attempting to load model with 8-bit quantization.")
else:
qwen2_audio_model = Qwen2AudioForConditionalGeneration.from_pretrained(
QWEN2_AUDIO_MODEL_NAME,
device_map="auto",
torch_dtype=torch.float32,
trust_remote_code=True
)
print("Loading model for CPU in float32.")
qwen2_audio_model.eval()
status_message = f"Qwen2-Audio model loaded on {device} successfully!"
print(status_message)
return status_message
except Exception as e:
error_message = f"Error loading Qwen2-Audio model: {e}"
print(error_message)
qwen2_audio_processor = None
qwen2_audio_model = None
clear_gpu_memory()
return f"Failed to load Qwen2-Audio model: {e}"
else:
status_message = "Qwen2-Audio model already loaded."
print(status_message)
return status_message
def create_qwen_invoice_prompt(audio_data_path: str, prompt_text: str = "") -> Dict:
"""
Creates the prompt for the Qwen2-Audio LLM, instructing it to extract structured invoice data.
The output format is explicitly requested as JSON.
Args:
audio_data_path (str): The local file path to the audio data.
This function assumes the audio is already downloaded locally.
prompt_text (str): Optional pre-provided transcript or additional prompt text.
Returns:
Dict: Inputs prepared for the Qwen2-Audio model.
"""
if qwen2_audio_processor is None:
raise RuntimeError("Qwen2-Audio processor is not loaded. Please load the model first.")
try:
# librosa expects a file path or file-like object. We are passing a local path.
audio, _ = librosa.load(audio_data_path, sr=qwen2_audio_processor.feature_extractor.sampling_rate)
except Exception as e:
raise ValueError(f"Error loading audio file from {audio_data_path}: {e}")
# Define the desired JSON schema for the LLM. Updated to match Pydantic model closely.
json_schema = """
{
"client_name": "string (e.g., John Doe, ACME Corp)",
"client_address": "string (e.g., 123 Main St, Anytown, CA)",
"invoice_number": "string (optional, e.g., INV-2025-001)",
"invoice_date": "YYYY-MM-DD (optional, defaults to today if not specified)",
"due_date": "YYYY-MM-DD (optional, defaults to invoice_date + 30 days if not specified)",
"items": [
{
"description": "string (e.g., Laptop, Consulting Services)",
"quantity": "float (e.g., 1.0, 2.5)",
"unit_price": "float (e.g., 1200.00, 75.50)"
}
],
"notes": "string (optional, any additional notes)"
}
"""
# Craft the prompt for the LLM, instructing it to act as an invoice data extractor.
messages = [
{"role": "system", "content": "You are an AI assistant that extracts structured invoice data from spoken requests. Your output MUST be a valid JSON object matching the provided schema. Do not include any other text or explanation outside the JSON. Extract all available details, and use best guesses for missing information (like current date for invoice_date)."},
{"role": "user", "content": f"Here is the invoice request. Please extract the details into a JSON object. Ensure item totals are calculated by quantity * unit_price. If no explicit invoice or due date, use today's date and 30 days from today respectively. If a field is not mentioned, omit it or set it to null. Here's the schema: {json_schema}\n\nInvoice request: {prompt_text}"}
]
# Qwen2-Audio expects a single audio input along with a list of messages
# The `audio` parameter is specifically for the audio input, not part of the messages content.
inputs = qwen2_audio_processor(audio=audio, text=messages, return_tensors="pt")
inputs = {k: v.to(device) for k, v in inputs.items()} # Move inputs to correct device
return inputs
def process_llm_output(output_ids: torch.Tensor, original_input_len: int) -> str:
"""Decodes the LLM output and extracts the JSON string."""
if qwen2_audio_processor is None:
raise RuntimeError("Qwen2-Audio processor is not loaded.")
# Decode the generated tokens
# Ensure to skip the original input tokens by slicing `output_ids`
generated_text = qwen2_audio_processor.decode(output_ids[0, original_input_len:], skip_special_tokens=True)
return generated_text
async def extract_and_validate_invoice_data(audio_object_name: str, transcript_text: str = "") -> InvoiceData:
"""
Downloads an audio file from MinIO, processes it with the LLM to extract invoice data,
validates it with Pydantic, and autofills missing information.
Args:
audio_object_name (str): The object name of the audio file in the MinIO audio bucket.
transcript_text (str): Optional pre-provided transcript of the audio.
Returns:
InvoiceData: The validated and autofilled Pydantic model representing the invoice.
"""
if qwen2_audio_model is None or qwen2_audio_processor is None:
raise RuntimeError("Qwen2-Audio model and/or processor not loaded. Call load_qwen2_audio_model() first.")
temp_audio_file_path = f"/tmp/{audio_object_name.split('/')[-1]}" # Use /tmp for temporary storage
try:
# 1. Download audio from MinIO to a temporary local file
audio_data_bytesio = minio_storage_service.download_file(MINIO_AUDIO_BUCKET, audio_object_name)
with open(temp_audio_file_path, "wb") as f:
f.write(audio_data_bytesio.read())
print(f"Downloaded audio from MinIO to temporary path: {temp_audio_file_path}")
# 2. Create prompt and get inputs for the LLM using the local file
llm_inputs = create_qwen_invoice_prompt(temp_audio_file_path, transcript_text)
original_input_len = llm_inputs['input_ids'].shape[1]
# 3. Generate response from LLM
with torch.no_grad():
generated_ids = qwen2_audio_model.generate(**llm_inputs, max_new_tokens=2048)
# 4. Process LLM output
llm_raw_output = process_llm_output(generated_ids, original_input_len)
print(f"LLM Raw Output:\n{llm_raw_output}")
# 5. Extract JSON part using regex
json_match = re.search(r"```json\s*(\{.*\})\s*```", llm_raw_output, re.DOTALL)
if not json_match:
# Fallback: try to find the JSON directly if it's not wrapped in markdown
json_match = re.search(r"(\{.*\})", llm_raw_output, re.DOTALL)
if not json_match:
raise ValueError("Could not extract a valid JSON object from LLM output.")
json_str = json_match.group(1)
try:
extracted_data = json.loads(json_str)
print(f"Extracted JSON Data: {extracted_data}")
# Validate and convert to Pydantic model
invoice_data = InvoiceData.model_validate(extracted_data)
print(f"Validated Invoice Data (before autofill): {invoice_data.model_dump_json(indent=2)}")
# Autofill missing details using the comprehensive function from utils
invoice_data = autofill_invoice_data(invoice_data)
print(f"Final Invoice Data (after autofill): {invoice_data.model_dump_json(indent=2)}")
return invoice_data
except json.JSONDecodeError as e:
raise ValueError(f"Failed to decode JSON from LLM output: {e}. Raw output: {json_str}")
except Exception as e: # Catch Pydantic ValidationError here and re-raise as ValueError
if "validation" in str(e).lower() and "model" in str(e).lower(): # Simple check for Pydantic validation error
raise ValueError(f"Pydantic validation failed for extracted data: {e}. Data: {json_str}")
else:
raise RuntimeError(f"An unexpected error occurred during data extraction or validation: {e}")
finally:
# Clean up the temporary audio file
if os.path.exists(temp_audio_file_path):
os.remove(temp_audio_file_path)
print(f"Cleaned up temporary audio file: {temp_audio_file_path}")

144
backend/services/pdf_service.py Normal file
View File

@@ -0,0 +1,144 @@
# backend/services/pdf_service.py
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Table, TableStyle
from reportlab.lib.styles import getSampleStyleSheet
from reportlab.lib.units import inch
from reportlab.lib import colors
from reportlab.lib.pagesizes import A4
# from reportlab.pdfgen import canvas # Not directly used with SimpleDocTemplate for main content
import os
from io import BytesIO
from datetime import datetime
from backend.models import InvoiceData
from backend.config import MINIO_PDF_BUCKET
from backend.services.storage_service import minio_storage_service # Import the MinIO service
def generate_invoice_pdf(invoice: InvoiceData) -> str:
"""
Generates a PDF invoice from the InvoiceData Pydantic model
and uploads it to the MinIO PDF bucket.
Args:
invoice (InvoiceData): The Pydantic model containing invoice data.
Returns:
str: The MinIO object name (path within the bucket) of the generated PDF.
"""
if not invoice.invoice_number:
# Generate a simple invoice number if not provided
invoice_number = f"INV-{invoice.invoice_date.replace('-', '') if invoice.invoice_date else datetime.now().strftime('%Y%m%d')}-{datetime.now().strftime('%H%M%S')}"
invoice.invoice_number = invoice_number
else:
invoice_number = invoice.invoice_number
# Sanitize client name for filename
sanitized_client_name = "".join(c for c in invoice.client_name if c.isalnum() or c in [' ', '_']).replace(' ', '_') if invoice.client_name else "unknown_client"
# Create a unique filename for the PDF in MinIO
pdf_object_name = f"invoice_{sanitized_client_name}_{invoice_number}_{datetime.now().strftime('%Y%m%d%H%M%S')}.pdf"
# Use BytesIO to create the PDF in memory
buffer = BytesIO()
doc = SimpleDocTemplate(buffer, pagesize=A4,
rightMargin=inch, leftMargin=inch,
topMargin=inch, bottomMargin=inch)
elements = []
styles = getSampleStyleSheet()
# --- Header ---
elements.append(Paragraph("<b>INVOICE</b>", styles['h1']))
elements.append(Spacer(1, 0.2 * inch))
elements.append(Paragraph(f"<b>Invoice #:</b> {invoice_number}", styles['Normal']))
elements.append(Paragraph(f"<b>Date:</b> {invoice.invoice_date if invoice.invoice_date else 'N/A'}", styles['Normal']))
elements.append(Paragraph(f"<b>Due Date:</b> {invoice.due_date if invoice.due_date else 'N/A'}", styles['Normal']))
elements.append(Spacer(1, 0.4 * inch))
# --- Client Information ---
elements.append(Paragraph("<b>Bill To:</b>", styles['h3']))
if invoice.client_name:
elements.append(Paragraph(invoice.client_name, styles['Normal']))
if invoice.client_address:
elements.append(Paragraph(invoice.client_address, styles['Normal']))
elements.append(Spacer(1, 0.4 * inch))
# --- Items Table ---
if invoice.items:
data = [['Description', 'Quantity', 'Unit Price', 'Total']]
for item in invoice.items:
# Ensure item totals are calculated if not present (though autofill_invoice_data handles this)
item_total = item.total if item.total is not None else (item.quantity * item.unit_price if item.quantity is not None and item.unit_price is not None else 0.0)
data.append([
item.description,
f"{item.quantity:.2f}",
f"${item.unit_price:.2f}",
f"${item_total:.2f}"
])
table_style = TableStyle([
('BACKGROUND', (0, 0), (-1, 0), colors.HexColor('#ADD8E6')), # Light blue header
('TEXTCOLOR', (0, 0), (-1, 0), colors.black),
('ALIGN', (0, 0), (-1, -1), 'LEFT'),
('FONTNAME', (0, 0), (-1, 0), 'Helvetica-Bold'),
('BOTTOMPADDING', (0, 0), (-1, 0), 12),
('BACKGROUND', (0, 1), (-1, -1), colors.HexColor('#F0F8FF')), # Alice blue rows
('GRID', (0, 0), (-1, -1), 1, colors.HexColor('#C0C0C0')), # Silver grid lines
('BOX', (0, 0), (-1, -1), 1, colors.HexColor('#808080')), # Grey box border
('ALIGN', (1, 0), (-1, -1), 'RIGHT'), # Align Qty, Unit Price, Total to right
])
item_table = Table(data, colWidths=[3*inch, 0.8*inch, 1*inch, 1*inch])
item_table.setStyle(table_style)
elements.append(item_table)
elements.append(Spacer(1, 0.2 * inch))
# --- Totals ---
# Ensure all totals are float and not None before formatting
subtotal = invoice.subtotal if invoice.subtotal is not None else 0.0
tax_amount = invoice.tax_amount if invoice.tax_amount is not None else 0.0
grand_total = invoice.grand_total if invoice.grand_total is not None else 0.0
tax_rate_display = invoice.tax_rate * 100 if invoice.tax_rate is not None else 0.0
totals_data = [
['Subtotal:', f"${subtotal:.2f}"],
[f"Tax ({tax_rate_display:.0f}%):", f"${tax_amount:.2f}"],
['Grand Total:', f"${grand_total:.2f}"]
]
totals_table = Table(totals_data, colWidths=[4.8*inch, 1.2*inch])
totals_table.setStyle(TableStyle([
('ALIGN', (0, 0), (-1, -1), 'RIGHT'),
('FONTNAME', (0, -1), (-1, -1), 'Helvetica-Bold'),
('TOPPADDING', (0, 0), (-1, -1), 4),
('BOTTOMPADDING', (0, 0), (-1, -1), 4),
]))
elements.append(totals_table)
elements.append(Spacer(1, 0.3 * inch))
# --- Notes ---
if invoice.notes:
elements.append(Paragraph("<b>Notes:</b>", styles['h3']))
elements.append(Paragraph(invoice.notes, styles['Normal']))
elements.append(Spacer(1, 0.2 * inch))
# --- Footer (Optional) ---
elements.append(Spacer(1, 0.5 * inch))
elements.append(Paragraph("<i>Thank you for your business!</i>", styles['Italic']))
# Build PDF in memory
try:
doc.build(elements)
buffer.seek(0) # Reset buffer position to the beginning
# Upload the PDF from memory to MinIO
minio_storage_service.upload_file(
bucket_name=MINIO_PDF_BUCKET,
object_name=pdf_object_name,
data=buffer,
length=buffer.tell(), # Get the current position (which is the file size)
content_type="application/pdf"
)
print(f"Invoice PDF generated and uploaded to MinIO: {MINIO_PDF_BUCKET}/{pdf_object_name}")
return pdf_object_name
except Exception as e:
print(f"Error generating or uploading PDF: {e}")
raise RuntimeError(f"Failed to generate or upload PDF: {e}")

145
backend/services/storage_service.py Normal file
View File

@@ -0,0 +1,145 @@
# backend/services/storage_service.py
import os
from io import BytesIO
from minio import Minio
from minio.error import S3Error
from backend.config import MINIO_ENDPOINT, MINIO_ACCESS_KEY, MINIO_SECRET_KEY, MINIO_SECURE, MINIO_AUDIO_BUCKET, MINIO_PDF_BUCKET
class MinIOStorageService:
"""
Service class for interacting with MinIO (S3-compatible) storage.
Handles uploading, downloading, and checking existence of objects in specified buckets.
"""
def __init__(self):
"""
Initializes the MinIO client.
Ensures the necessary buckets exist on startup.
"""
try:
self.client = Minio(
endpoint=MINIO_ENDPOINT,
access_key=MINIO_ACCESS_KEY,
secret_key=MINIO_SECRET_KEY,
secure=MINIO_SECURE,
# Set a higher timeout for potentially large file uploads/downloads
# or slow network conditions. Default is usually sufficient but good for large files.
# Timeout in seconds. Can be None for no timeout.
http_client=None # Use default http client, or provide custom
)
print(f"MinIO client initialized for endpoint: {MINIO_ENDPOINT}, Secure: {MINIO_SECURE}")
self._ensure_buckets_exist()
except Exception as e:
print(f"ERROR: Failed to initialize MinIO client: {e}")
raise
def _ensure_buckets_exist(self):
"""
Ensures that the audio and PDF buckets exist. Creates them if they don't.
"""
buckets_to_create = [MINIO_AUDIO_BUCKET, MINIO_PDF_BUCKET]
for bucket_name in buckets_to_create:
try:
if not self.client.bucket_exists(bucket_name):
self.client.make_bucket(bucket_name)
print(f"MinIO bucket '{bucket_name}' created successfully.")
else:
print(f"MinIO bucket '{bucket_name}' already exists.")
except S3Error as e:
print(f"ERROR: S3 Error ensuring bucket '{bucket_name}': {e}")
raise
except Exception as e:
print(f"ERROR: Unexpected error ensuring bucket '{bucket_name}': {e}")
raise
def upload_file(self, bucket_name: str, object_name: str, data: BytesIO, length: int, content_type: str = "application/octet-stream") -> str:
"""
Uploads a file (BytesIO object) to a specified MinIO bucket.
Args:
bucket_name (str): The name of the bucket.
object_name (str): The desired name of the object in the bucket.
data (BytesIO): The file data as a BytesIO object.
length (int): The length of the data in bytes.
content_type (str): The MIME type of the file.
Returns:
str: The full path of the uploaded object (bucket_name/object_name).
"""
try:
self.client.put_object(
bucket_name=bucket_name,
object_name=object_name,
data=data,
length=length,
content_type=content_type
)
print(f"Successfully uploaded {object_name} to bucket {bucket_name}")
# Construct a downloadable URL (for MinIO, this usually implies access via its API/proxy)
# For direct public access, you'd need presigned URLs or public buckets configured externally.
# Here, we return a logical path that your FastAPI can use to retrieve.
return f"{bucket_name}/{object_name}"
except S3Error as e:
print(f"ERROR: S3 Error uploading {object_name} to {bucket_name}: {e}")
raise
except Exception as e:
print(f"ERROR: Unexpected error uploading {object_name}: {e}")
raise
def download_file(self, bucket_name: str, object_name: str) -> BytesIO:
"""
Downloads a file from a specified MinIO bucket.
Args:
bucket_name (str): The name of the bucket.
object_name (str): The name of the object to download.
Returns:
BytesIO: The file data as a BytesIO object.
"""
try:
response = self.client.get_object(bucket_name, object_name)
file_data = BytesIO(response.read())
file_data.seek(0) # Reset stream position to the beginning
response.close()
response.release_conn()
print(f"Successfully downloaded {object_name} from bucket {bucket_name}")
return file_data
except S3Error as e:
if e.code == "NoSuchKey":
print(f"ERROR: Object '{object_name}' not found in bucket '{bucket_name}'.")
raise FileNotFoundError(f"Object '{object_name}' not found in bucket '{bucket_name}'.")
else:
print(f"ERROR: S3 Error downloading {object_name} from {bucket_name}: {e}")
raise
except Exception as e:
print(f"ERROR: Unexpected error downloading {object_name}: {e}")
raise
def object_exists(self, bucket_name: str, object_name: str) -> bool:
"""
Checks if an object exists in a specified MinIO bucket.
Args:
bucket_name (str): The name of the bucket.
object_name (str): The name of the object to check.
Returns:
bool: True if the object exists, False otherwise.
"""
try:
self.client.stat_object(bucket_name, object_name)
return True
except S3Error as e:
if e.code == "NoSuchKey":
return False
else:
print(f"ERROR: S3 Error checking existence of {object_name} in {bucket_name}: {e}")
raise
except Exception as e:
print(f"ERROR: Unexpected error checking existence of {object_name}: {e}")
raise
# Instantiate the service globally or pass it around via dependency injection in FastAPI
# For simplicity, we'll instantiate it here. FastAPI's Depends can be used for DI.
minio_storage_service = MinIOStorageService()