helper_tables.py

from azure.identity import DefaultAzureCredential
from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient, BlobBlock
import azure.functions as func
import pandas as pd
import tempfile
from config import LOCAL 

def generate_helper_table(file_name_azure, df, blob_service_client):
    fp_file = tempfile.NamedTemporaryFile()
    if LOCAL:
        try:
            with open(file_name_azure, 'r', encoding='utf-8') as f:
                data = f.read()
                fp_file.write(data.encode('utf-8'))
        except FileNotFoundError as e:
            return df
    else:
        try:
            blob = blob_service_client.get_blob_client(container='helper-data', blob=file_name_azure)
            fp_file.write(blob.download_blob().readall())
        except Exception as e:
            return df
    fp_file.seek(0)
    old_df = pd.read_csv(fp_file, header=0)
    fp_file.close()

    sub = get_subset(file_name_azure)
    concat = pd.concat([old_df,df])
    wodups = concat.drop_duplicates(subset=sub, keep='last')
    return wodups

def get_subset(name):
    if name == 'helper_cases.csv':
        return ['ride_id']
    elif name == 'helper_tracks.csv':
        return ['line', 'name', 'direction']
    elif name == 'helper_stops.csv':
        return ['bus_stop_id']
    elif name == 'helper_days.csv':
        return ['day_code']
    elif name == 'helper_cities.csv':
        return ['city_code']
    else:
        return ['']

def get_helper_blob(file_name, blob_service_client, day_raw):
    split = file_name.split('_', 1)
    month = day_raw[:-2]
    try:
        if split[0].isdigit():
            return blob_service_client.get_blob_client(container='helper-data', blob=f'{month}/{day_raw}/{file_name}')
        else:
            return blob_service_client.get_blob_client(container='helper-data', blob=file_name)
    except Exception as e:
        print(e)
        return None