adding download script, goldens and validation_config, updated mapping for latest data

statvar_imports/us_nces/nces_employed_college_grads/download.py

@@ -0,0 +1,167 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#           https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import re
+from urllib.parse import urlparse
+from absl import app
+from absl import logging
+import openpyxl
+import requests
+
+# Add data/util to sys.path so we can import the shared wrapper functions
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+PROJECT_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, '../../..'))
+UTIL_DIR = os.path.join(PROJECT_ROOT, 'util')
+if UTIL_DIR not in sys.path:
+    sys.path.insert(0, UTIL_DIR)
+
+try:
+    from download_util_script import download_file, _retry_method
+except ImportError:
+    logging.fatal(
+        "Could not import download_file from 'util/download_util_script.py'."
+    )
+    sys.exit(1)
+
+LANDING_PAGE_URL = "https://ncses.nsf.gov/surveys/national-survey-college-graduates"
+FILE_PATTERN = r'/pubs/[^/]+/assets/data-tables/tables/[^/]*tab006-002\.(?:xlsx|csv)'
+OUTPUT_FOLDER = os.path.join(SCRIPT_DIR, "source_files")
+YEAR_HEADER_PATTERN = re.compile(r'^\s*(\d{4})[a-zA-Z*#]+\s*$')
+
+
+def resolve_url(landing_url=LANDING_PAGE_URL,
+                file_pattern=FILE_PATTERN,
+                headers=None,
+                tries=10,
+                delay=5,
+                backoff=2):
+    """
+    Scrapes landing page HTML to dynamically find matching table URL.
+
+    Args:
+        landing_url: URL of the webpage containing table links.
+        file_pattern: Regex pattern to match the target link.
+        headers: Optional dictionary of HTTP headers to send with the request.
+        tries: Number of retry attempts.
+        delay: Initial delay for retries.
+        backoff: Backoff factor for retries.
+
+    Returns:
+        str: Absolute URL of the target file, or None if not found.
+    """
+    logging.info(f"Attempting to resolve target URL from landing page: {landing_url}")
+
+    try:
+        response = _retry_method(landing_url, headers, tries, delay, backoff)
+        response.raise_for_status()
+    except (requests.exceptions.RequestException, ValueError, OSError) as e:
+        logging.error(f"Failed to fetch landing page '{landing_url}': {e}")
+        return None
+    except Exception as e:
+        logging.fatal(
+            f"An unexpected error occurred while fetching landing page '{landing_url}': {e}"
+        )
+        return None
+
+    matches = re.findall(file_pattern, response.text)
+    if not matches:
+        logging.error(
+            f"No link matching pattern '{file_pattern}' found on '{landing_url}'."
+        )
+        return None
+
+    resolved_path = matches[0]
+    parsed_landing = urlparse(landing_url)
+    base_domain = f"{parsed_landing.scheme}://{parsed_landing.netloc}"
+    resolved_url = f"{base_domain}{resolved_path}"
+    logging.info(f"Dynamically resolved download URL: {resolved_url}")
+    return resolved_url
+
+
+def clean_year_headers(folder_path, max_header_row=4):
+    """
+    Cleans year headers in downloaded Excel files by removing footnote suffixes
+    (e.g., '2023a' -> '2023') strictly in the top header rows (rows 1-4).
+    Leaves all data rows (row 5+) completely untouched.
+
+    Args:
+        folder_path: Path to the directory containing downloaded Excel files.
+        max_header_row: Maximum row index to inspect for header columns.
+
+    Returns:
+        bool: True if header cleaning succeeded, False if an error occurred.
+    """
+    if not folder_path or not os.path.exists(folder_path):
+        return True
+
+    for filename in os.listdir(folder_path):
+        if not filename.endswith('.xlsx'):
+            continue
+
+        file_path = os.path.join(folder_path, filename)
+        wb = None
+        try:
+            wb = openpyxl.load_workbook(file_path)
+            sheet = wb.active
+            modified = False
+            for row in sheet.iter_rows(max_row=max_header_row):
+                for cell in row:
+                    if isinstance(cell.value, str):
+                        new_val = YEAR_HEADER_PATTERN.sub(r'\1', cell.value)
+                        if new_val != cell.value:
+                            cell.value = new_val
+                            modified = True
+            if modified:
+                wb.save(file_path)
+                logging.info(f"Successfully cleaned year headers in '{file_path}'")
+        except (ValueError, OSError) as e:
+            logging.error(f"Error cleaning headers in file '{file_path}': {e}")
+            return False
+        except Exception as e:
+            logging.fatal(
+                f"An unexpected error occurred while cleaning headers in '{file_path}': {e}"
+            )
+            return False
+        finally:
+            if wb is not None:
+                wb.close()
+
+    return True
+
+
+def main(_):
+    logging.set_verbosity(logging.INFO)
+    logging.info("Script execution started...")
+
+    resolved_url = resolve_url(LANDING_PAGE_URL, FILE_PATTERN, None)
+    if not resolved_url:
+        logging.error("Failed to resolve URL from landing page.")
+        sys.exit(1)
+
+    if not download_file(resolved_url, OUTPUT_FOLDER, False, None, tries=10):
+        logging.error(
+            "File download or processing failed. Check logs for details.")
+        sys.exit(1)
+
+    if not clean_year_headers(OUTPUT_FOLDER):
+        logging.error("Year header cleaning failed. Check logs for details.")
+        sys.exit(1)
+
+    logging.info("Script processing completed successfully.")
+
+
+if __name__ == '__main__':
+    app.run(main)

statvar_imports/us_nces/nces_employed_college_grads/golden_data/golden_observations.csv

@@ -0,0 +1,2 @@
+"observationAbout"
+"country/USA"

statvar_imports/us_nces/nces_employed_college_grads/golden_data/golden_summary_report.csv

@@ -0,0 +1,20 @@
+"ScalingFactors","MinDate","MeasurementMethods","observationPeriods","StatVar","Units","NumPlaces"
+"[]","2003","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_SOCPhysicalScientistsOccupation_WhiteAlone","[]","1"
+"[]","2003","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_HispanicOrLatino_SOCSocialScientistsRelatedWorkersOccupation","[]","1"
+"[]","2003","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_SOCPhysicalScientistsOccupation_AmericanIndianOrAlaskaNative","[]","1"
+"[]","2003","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_SOCLifeScientistsOccupation_AmericanIndianOrAlaskaNative","[]","1"
+"[]","2003","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_SOCPhysicalScientistsOccupation_OtherPacificIslander","[]","1"
+"[]","2003","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_HispanicOrLatino_SOCPhysicalScientistsOccupation","[]","1"
+"[]","2003","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_HispanicOrLatino","[]","1"
+"[]","2003","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_Male_SOCEngineersOccupation","[]","1"
+"[]","2015","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_SOCLifeScientistsOccupation_Asian","[]","1"
+"[]","2003","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_SOCSocialScientistsRelatedWorkersOccupation_WhiteAlone","[]","1"
+"[]","2003","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_SOCLifeScientistsOccupation_OtherPacificIslander","[]","1"
+"[]","2003","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_SOCLifeScientistsOccupation_WhiteAlone","[]","1"
+"[]","2003","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_Female_SOCEngineersOccupation","[]","1"
+"[]","2003","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_HispanicOrLatino_SOCLifeScientistsOccupation","[]","1"
+"[]","2003","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_SOCComputerMathematicalOccupation_AmericanIndianOrAlaskaNative","[]","1"
+"[]","2003","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_SOCSocialScientistsRelatedWorkersOccupation_OtherPacificIslander","[]","1"
+"[]","2013","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_SOCSocialScientistsRelatedWorkersOccupation_AmericanIndianOrAlaskaNative","[]","1"
+"[]","2003","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_HispanicOrLatino_SOCComputerMathematicalOccupation","[]","1"
+"[]","2019","[]","[]","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher_Employed_SOCComputerMathematicalOccupation_OtherPacificIslander","[]","1"

statvar_imports/us_nces/nces_employed_college_grads/manifest.json

@@ -8,11 +8,12 @@
             "provenance_url": "https://ncsesdata.nsf.gov/explorer/datatables?term=race&exactMatch=no&page=1&filterSuperTopic=Demographics&filterTopic=Sex&datatablespage=2",
             "provenance_description": "NCSES is the U.S. agency that collects and reports data on science, engineering, and technology",
             "scripts": [
-                "../../../util/download_util_script.py --download_url=https://ncses.nsf.gov/pubs/nsf23306/assets/data-tables/tables/nsf23306-tab006-002.xlsx --output_folder=source_files",
-                "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.xlsx --pv_map=pv_map.csv --config_file=metadata.csv --output_path=output/nces_college"
+                "download.py",
+                "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.xlsx --pv_map=pv_map.csv --config_file=metadata.csv --output_path=output/nces_college --output_counters=counters/nces_college.csv"
             ],
             "source_files": [
-                "source_files/*.xlsx"
+                "source_files/*.xlsx",
+                "counters/*.csv"
             ],
             "import_inputs": [
                 {

statvar_imports/us_nces/nces_employed_college_grads/pv_map.csv

@@ -1,38 +1,42 @@
-key,p1,v1,p2,v2,p3,v3,p4,v4,p5,v5,p6,v6,p7,v7
-2003,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2003,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed
-2010,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2010,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed
-2013,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2013,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed
-2015,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2015,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed
-2017,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2017,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed
-2019,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2019,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed
-2021,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2021,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed
-2022,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2022,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed
-2023,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2023,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed
-2024,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2024,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed
-2025,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2025,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed
-2026,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2026,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed
-2027,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2027,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed
-2028,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2028,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed
-2029,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2029,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed
-2030,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2030,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed
-Male,gender,Male,race,"""""",ethnicity,"""""",,,,,,,,
-Female,gender,Female,race,"""""",ethnicity,"""""",,,,,,,,
-Both sexes,gender,"""""",race,"""""",ethnicity,"""""",,,,,,,,
-White,race,WhiteAlone,ethnicity,"""""",,,,,,,,,,
-Black or African American,race,BlackOrAfricanAmericanAlone,ethnicity,"""""",,,,,,,,,,
-Hispanic or Latino,ethnicity,HispanicOrLatino,race,"""""",,,,,,,,,,
-Asian,race,Asian,ethnicity,"""""",,,,,,,,,,
-American Indian or Alaska Native,race,AmericanIndianOrAlaskaNative,ethnicity,"""""",,,,,,,,,,
-Native Hawaiian or Other Pacific Islander,race,OtherPacificIslander,ethnicity,"""""",,,,,,,,,,
-More than one race,race,TwoOrMoreRaces,ethnicity,"""""",,,,,,,,,,
+key,p1,v1,p2,v2,p3,v3,p4,v4,p5,v5,p6,v6,p7,v7,p8,v8
+2003,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2003,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed,,
+2010,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2010,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed,,
+2013,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2013,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed,,
+2015,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2015,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed,,
+2017,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2017,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed,,
+2019,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2019,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed,,
+2021,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2021,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed,,
+2022,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2022,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed,,
+2023,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2023,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed,,
+2024,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2024,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed,,
+2025,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2025,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed,,
+2026,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2026,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed,,
+2027,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2027,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed,,
+2028,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2028,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed,,
+2029,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2029,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed,,
+2030,populationType,Person,measuredProperty,count,value,@Number,observationAbout,country/USA,observationDate,2030,educationalAttainment,BachelorsDegreeOrHigher,employmentStatus,Employed,,
+Male,gender,Male,race,"""""",ethnicity,"""""",#Header,"gender,race,ethnicity",,,
+Female,gender,Female,race,"""""",ethnicity,"""""",#Header,"gender,race,ethnicity",,,
+Both sexes,gender,"""""",race,"""""",ethnicity,"""""",#Header,"gender,race,ethnicity",,,
+Hispanic or Latino,ethnicity,HispanicOrLatino,race,"""""",#Header,"ethnicity,race",,,,,
+Not Hispanic or Latino,ethnicity,NotHispanicOrLatino,race,"""""",#Header,"ethnicity,race",,,,,
+White,race,WhiteAlone,ethnicity,NotHispanicOrLatino,#Header,"race,ethnicity",,,,,
+Black or African American,race,BlackOrAfricanAmericanAlone,ethnicity,NotHispanicOrLatino,#Header,"race,ethnicity",,,,,
+Asian,race,Asian,ethnicity,NotHispanicOrLatino,#Header,"race,ethnicity",,,,,
+American Indian or Alaska Native,race,AmericanIndianOrAlaskaNative,ethnicity,NotHispanicOrLatino,#Header,"race,ethnicity",,,,,
+Native Hawaiian or Other Pacific Islander,race,OtherPacificIslander,ethnicity,NotHispanicOrLatino,#Header,"race,ethnicity",,,,,
+More than one race,race,TwoOrMoreRaces,ethnicity,NotHispanicOrLatino,#Header,"race,ethnicity",,,,,
+Biological, agricultural, and other life scientists,occupation,SOCv2018/19-1000,,,,,,,,,,,,
 "Biological, agricultural, and other life scientists",occupation,SOCv2018/19-1000,,,,,,,,,,,,
 Computer and mathematical scientists,occupation,SOCv2018/15-0000,,,,,,,,,,,,
 Physical and related scientists,occupation,SOCv2018/19-2000,,,,,,,,,,,,
 Social and related scientists,occupation,SOCv2018/19-3000,,,,,,,,,,,,
 Engineers,occupation,SOCv2018/17-2000,,,,,,,,,,,,
+S&E occupations,occupation,ScienceAndEngineering,,,,,,,,,,,,
 All S&E occupations,occupation,ScienceAndEngineering,,,,,,,,,,,,
+S&E-related occupations,occupation,ScienceAndEngineeringRelated,,,,,,,,,,,,
 All S&E-related occupations,occupation,ScienceAndEngineeringRelated,,,,,,,,,,,,
-,,,,,,,,,,,,,,
+Non-S&E occupations,occupation,NonScienceAndEngineering,,,,,,,,,,,,
 All Non-S&E occupations,occupation,NonScienceAndEngineering,,,,,,,,,,,,
 D,value,0,#ignore,suppressed,,,,,,,,,,
 S,value,0,#ignore,suppressed,,,,,,,,,,