Update app.py

app.py

@@ -66,28 +66,34 @@ def load_and_concat_data():
         try:
             file_content = api.hf_hub_download(repo_id=f"{HF_USERNAME}/{DATASET_NAME}", filename=file, repo_type="dataset", token=HF_TOKEN)
             
-            # Use CSV sniffer to detect delimiter and number of columns
-            with open(file_content, 'r') as f:
-                sample = f.read(1024)
-                sniffer = csv.Sniffer()
-                dialect = sniffer.sniff(sample)
-                f.seek(0)
-                
-                # Use pyarrow for more flexible parsing
-                parse_options = csv.ParseOptions(delimiter=dialect.delimiter)
-                table = csv.read_csv(file_content, parse_options=parse_options)
-                
-                # Convert to pandas DataFrame
-                df = table.to_pandas()
+            # Use PyArrow's CSV reading capabilities
+            read_options = csv.ReadOptions(use_threads=True)
+            parse_options = csv.ParseOptions(delimiter=',')  # Adjust delimiter if needed
+            convert_options = csv.ConvertOptions(
+                column_types={
+                    'date_posted': pa.timestamp('s'),
+                    'is_remote': pa.bool_()
+                },
+                strings_can_be_null=True
+            )
             
-            # Ensure all required columns are present, fill with NaN if missing
-            required_columns = ['site', 'job_url', 'title', 'company', 'location', 'job_type', 'date_posted', 'is_remote', 'company_url']
-            for col in required_columns:
-                if col not in df.columns:
-                    df[col] = pd.NA
+            table = csv.read_csv(file_content, read_options=read_options, parse_options=parse_options, convert_options=convert_options)
+            df = table.to_pandas()
             
-            # Select only the required columns
-            df = df[required_columns]
+            # Perform data cleaning and processing
+            df['date_posted'] = pd.to_datetime(df['date_posted'], errors='coerce')
+            df = df.dropna(subset=['date_posted'])
+            df = df[df['date_posted'].dt.year == 2024]
+            df['title'] = df['title'].str.lower()
+            df['company'] = df['company'].str.lower()
+            
+            def clean_location(location):
+                if pd.isna(location):
+                    return location
+                location = str(location).lower()
+                return re.sub(r',\s*(us|usa)$', '', location)
+            
+            df['location'] = df['location'].apply(clean_location)
             
             all_data.append(df)
         except Exception as e:
@@ -98,24 +104,9 @@ def load_and_concat_data():
         return pd.DataFrame()
 
     concatenated_df = pd.concat(all_data, ignore_index=True)
-    
-    # Perform data cleaning and processing
-    concatenated_df['date_posted'] = pd.to_datetime(concatenated_df['date_posted'], errors='coerce')
-    concatenated_df = concatenated_df.dropna(subset=['date_posted'])
-    concatenated_df = concatenated_df[concatenated_df['date_posted'].dt.year == 2024]
-    concatenated_df['title'] = concatenated_df['title'].str.lower()
-    concatenated_df['company'] = concatenated_df['company'].str.lower()
-    
-    def clean_location(location):
-        if pd.isna(location):
-            return location
-        location = str(location).lower()
-        return re.sub(r',\s*(us|usa)$', '', location)
-
-    concatenated_df['location'] = concatenated_df['location'].apply(clean_location)
-    concatenated_df = concatenated_df.drop_duplicates()
-    
-    return concatenated_df
+    filtered_df = concatenated_df.drop_duplicates().reset_index(drop=True)
+
+    return filtered_df
     
 @st.cache_data()
 def get_unique_values(df):