import pandas as pd
import os

def filter_parquet_files():
    source_dir = './test/testtool/source'
    output_file = './test/testtool/filtered_output.snappy.parquet'
    
    filtered_dfs = []
    print(os.getcwd())
    # Iterate through all Parquet files in the source directory
    for filename in os.listdir(source_dir):
        if filename.endswith('.parquet'):
            file_path = os.path.join(source_dir, filename)
            print(f"Processing file: {file_path}")
            
            # read parquet file
            df = pd.read_parquet(file_path)
            # convert usage_date to date
            df['usage_date'] = pd.to_datetime(df['line_item_usage_start_date'], unit='s').dt.date
            target_date = pd.to_datetime('2024-12-21').date()
            
            # filter rows
            filtered_df = df[
                (df['resource_tags_user_tenant'] == '60205') &
                (df['usage_date'] == target_date)
            ]
            filtered_dfs.append(filtered_df)
    
    # combine all filtered dataframes
    if filtered_dfs:
        combined_df = pd.concat(filtered_dfs, ignore_index=True)
        print(f"Writing combined data to {output_file}")
        
        # write to new parquet file
        combined_df.to_parquet(output_file, compression='snappy')
    else:
        print("No matching data found.")

def main():
    filter_parquet_files()

if __name__ == "__main__":
    main()