from langchain.llms import AzureOpenAI
from langchain.prompts import PromptTemplate
from crewai import Agent, Task, Crew
from azure.storage.blob import BlobServiceClient
import os

# Cấu hình Azure Blob
blob_service_client = BlobServiceClient.from_connection_string(os.getenv("AZURE_STORAGE_CONNECTION_STRING"))
container_client = blob_service_client.get_container_client("data-pipeline")

# Khởi tạo LangChain LLM
llm = AzureOpenAI(
    deployment_name="gpt-4",
    api_key=os.getenv("AZURE_OPENAI_API_KEY"),
    api_version="2023-05-15"
)

transform_prompt = PromptTemplate(
    input_variables=["data"],
    template="Clean and transform this JSON data: {data}. Handle missing values and standardize formats."
)

# Khởi tạo AI agents CrewAI
data_ingestion_agent = Agent(
    role="Data Ingester",
    goal="Ingest raw data into Azure Blobs",
    backstory="Expert in data extraction and cloud storage.",
    llm=llm
)

data_transform_agent = Agent(
    role="Data Transformer",
    goal="Transform data using LangChain",
    backstory="Skilled in data cleansing and enrichment.",
    llm=llm
)

# Định nghĩa các task
ingestion_task = Task(
    description="Ingest raw sales data from Kafka and upload to Azure Blobs.",
    agent=data_ingestion_agent,
    callback=lambda result: container_client.upload_blob("raw/sales.json", result, overwrite=True)
)

transform_task = Task(
    description="Download raw data from Azure Blobs, transform using LangChain, and upload processed data.",
    agent=data_transform_agent,
    callback=lambda result: container_client.upload_blob("processed/sales.json", result, overwrite=True)
)

# Khởi động pipeline CrewAI
crew = Crew(
    agents=[data_ingestion_agent, data_transform_agent],
    tasks=[ingestion_task, transform_task],
    verbose=True
)

crew.kickoff()

# Thực thi bước biến đổi dữ liệu với LangChain
blob_client = container_client.get_blob_client("raw/sales.json")
raw_data = blob_client.download_blob().readall().decode("utf-8")
chain = transform_prompt | llm
transformed_data = chain.invoke({"data": raw_data})

# Tải lên dữ liệu đã biến đổi
container_client.upload_blob("processed/sales.json", transformed_data, overwrite=True)