Skip to content

Commit 338ffbf

Browse files
Charlie-HenryCharlie-Henry
authored
Merge pull request #28 from cityofaustin/charlie/program_tagging
FDU program tagging
2 parents dcebed2 + 386514b commit 338ffbf

File tree

3 files changed

+275
-1
lines changed

3 files changed

+275
-1
lines changed

‎program_tagging/fdu_program_tagging.py‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import os
2+
import json
3+
import logging
4+
5+
import boto3
6+
import sodapy
7+
8+
import utils
9+
from prefix_maps import prefix_mapping
10+
11+
AWS_ACCESS_ID = os.getenv("AWS_ACCESS_KEY_ID")
12+
AWS_PASS = os.getenv("AWS_SECRET_ACCESS_KEY")
13+
BUCKET_NAME = os.getenv("BUCKET")
14+
15+
SO_WEB = os.getenv("SO_WEB")
16+
SO_TOKEN = os.getenv("SO_TOKEN")
17+
SO_USER = os.getenv("SO_USER")
18+
SO_PASS = os.getenv("SO_PASS")
19+
20+
DATASET = os.getenv("PROGRAM_DATASET")
21+
22+
23+
def get_data(client):
24+
"""
25+
Gets the subprojects.json file and sends the data to socrata
26+
27+
Parameters
28+
----------
29+
client : AWS Client object
30+
31+
socrata_client : Socrata client object
32+
33+
Returns
34+
-------
35+
None.
36+
37+
"""
38+
response = client.get_object(Bucket=BUCKET_NAME, Key="fdu_expenses_obligated.json")
39+
obj_data = response.get("Body").read().decode()
40+
data = json.loads(obj_data)
41+
return data
42+
43+
44+
def tag_programs(data):
45+
for row in data:
46+
row["id"] = (
47+
row["DEPT"] + row["FUND"] + row["LVL1_DIV_CODE"] + row["LVL2_GP_CODE"]
48+
)
49+
for prefix in prefix_mapping:
50+
if row["id"].startswith(prefix):
51+
row["program"] = prefix_mapping[prefix]["program"]
52+
row["subprogram"] = prefix_mapping[prefix]["subprogram"]
53+
row["bond_year"] = prefix_mapping[prefix]["bond_year"]
54+
return data
55+
56+
57+
def main():
58+
aws_s3_client = boto3.client(
59+
"s3",
60+
aws_access_key_id=AWS_ACCESS_ID,
61+
aws_secret_access_key=AWS_PASS,
62+
)
63+
socrata_client = sodapy.Socrata(
64+
SO_WEB,
65+
SO_TOKEN,
66+
username=SO_USER,
67+
password=SO_PASS,
68+
timeout=60,
69+
)
70+
71+
data = get_data(aws_s3_client)
72+
data = tag_programs(data)
73+
res = socrata_client.replace(DATASET, data)
74+
print(res)
75+
76+
77+
if __name__ == "__main__":
78+
logger = utils.get_logger(__name__, level=logging.INFO)
79+
80+
main()

‎program_tagging/prefix_maps.py‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
prefix_mapping = {
2+
"25078119C001": {
3+
"program": "Corridor Program",
4+
"subprogram": "Corridor Program",
5+
"bond_year": 2016,
6+
},
7+
"25078119L001L010": {
8+
"program": "Bikeways",
9+
"subprogram": "Bikeways",
10+
"bond_year": 2016,
11+
},
12+
"25078119L001L030": {
13+
"program": "Vision Zero",
14+
"subprogram": "Intersection Safety",
15+
"bond_year": 2016,
16+
},
17+
"25078119L001L100": {
18+
"program": "Substandard Streets",
19+
"subprogram": "Substandard Streets",
20+
"bond_year": 2016,
21+
},
22+
"25078119R001": {
23+
"program": "Regional Mobility",
24+
"subprogram": "Regional Mobility",
25+
"bond_year": 2016,
26+
},
27+
"62078119L001": {
28+
"program": "Sidewalks",
29+
"subprogram": "Sidewalks",
30+
"bond_year": 2016,
31+
},
32+
"62078119L100": {
33+
"program": "Safe Routes",
34+
"subprogram": "Safe Routes",
35+
"bond_year": 2016,
36+
},
37+
"62078119L200": {
38+
"program": "Urban Trails",
39+
"subprogram": "Urban Trails",
40+
"bond_year": 2016,
41+
},
42+
"62078119L300": {
43+
"program": "Capital Renewal",
44+
"subprogram": "Capital Renewal",
45+
"bond_year": 2016,
46+
},
47+
"25078127B100": {
48+
"program": "Traffic Signals",
49+
"subprogram": "Traffic Signals",
50+
"bond_year": 2018,
51+
},
52+
"25078127B110": {
53+
"program": "Vision Zero",
54+
"subprogram": "Intersection Safety",
55+
"bond_year": 2018,
56+
},
57+
"25078127B140": {
58+
"program": "Vision Zero",
59+
"subprogram": "Pedestrian Safety",
60+
"bond_year": 2018,
61+
},
62+
"62078127B001": {
63+
"program": "Street Programs",
64+
"subprogram": "Street Programs",
65+
"bond_year": 2018,
66+
},
67+
"62078127B100": {
68+
"program": "Bridges & Structures",
69+
"subprogram": "Bridges & Structures",
70+
"bond_year": 2018,
71+
},
72+
"62078127B200": {
73+
"program": "Sidewalks",
74+
"subprogram": "Sidewalks",
75+
"bond_year": 2018,
76+
},
77+
"62078127B300": {
78+
"program": "Urban Trails",
79+
"subprogram": "Urban Trails",
80+
"bond_year": 2018,
81+
},
82+
"62078127B400": {
83+
"program": "Neighborhood Partnering",
84+
"subprogram": "Neighborhood Partnering",
85+
"bond_year": 2018,
86+
},
87+
"2507820BD001": {
88+
"program": "Bikeways",
89+
"subprogram": "Bikeways",
90+
"bond_year": 2020,
91+
},
92+
"2507820BD003D300": {
93+
"program": "Vision Zero",
94+
"subprogram": "Signal Safety",
95+
"bond_year": 2020,
96+
},
97+
"2507820BD003D400": {
98+
"program": "Vision Zero",
99+
"subprogram": "Pedestrian Safety",
100+
"bond_year": 2020,
101+
},
102+
"2507820BD003D500": {
103+
"program": "Vision Zero",
104+
"subprogram": "Major Safety",
105+
"bond_year": 2020,
106+
},
107+
"2507820BD003D600": {
108+
"program": "Vision Zero",
109+
"subprogram": "Speed Management",
110+
"bond_year": 2020,
111+
},
112+
"2507820BD007D700": {
113+
"program": "Local Transit",
114+
"subprogram": "Local Transit Enhancement",
115+
"bond_year": 2020,
116+
},
117+
"2507820BD007D730": {
118+
"program": "Local Transit",
119+
"subprogram": "Signal Priority",
120+
"bond_year": 2020,
121+
},
122+
"2507820BD007D740": {
123+
"program": "Local Transit",
124+
"subprogram": "Signal Cabinet Security",
125+
"bond_year": 2020,
126+
},
127+
"2507820BD007D750": {
128+
"program": "Local Transit",
129+
"subprogram": "Micro Mobility",
130+
"bond_year": 2020,
131+
},
132+
"2507820BD007D760": {
133+
"program": "Local Transit",
134+
"subprogram": "Smart Mobility",
135+
"bond_year": 2020,
136+
},
137+
"2507820BD008": {
138+
"program": "Substandard Streets",
139+
"subprogram": "Substandard Streets",
140+
"bond_year": 2020,
141+
},
142+
"2507820BD009": {
143+
"program": "Capital Renewal",
144+
"subprogram": "Capital Renewal",
145+
"bond_year": 2020,
146+
},
147+
"6207820BD000": {
148+
"program": "Sidewalks",
149+
"subprogram": "Sidewalks",
150+
"bond_year": 2020,
151+
},
152+
"6207820BDCI0": {
153+
"program": "Capital Renewal",
154+
"subprogram": "Capital Renewal",
155+
"bond_year": 2020,
156+
},
157+
"6207820BDN00": {
158+
"program": "Neighborhood Partnering",
159+
"subprogram": "Neighborhood Partnering",
160+
"bond_year": 2020,
161+
},
162+
"6207820BDSR0": {
163+
"program": "Safe Routes",
164+
"subprogram": "Safe Routes",
165+
"bond_year": 2020,
166+
},
167+
"6207820BDT00": {
168+
"program": "Urban Trails",
169+
"subprogram": "Urban Trails",
170+
"bond_year": 2020,
171+
},
172+
}

‎queries.py‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@
9393
ATD_SUBPROJECT_FDU_VW
9494
""",
9595
"subprojects": """
96-
SELECT
96+
SELECT
9797
PROJECT_NUMBER,
9898
SP_NUMBER_TXT,
9999
SP_NAME,
@@ -105,4 +105,26 @@
105105
FROM
106106
MSTR_IA_DEV.DEPT_2400_SUBPRJ_VW
107107
""",
108+
"fdu_expenses_obligated": """
109+
SELECT
110+
FDU,
111+
FDU_STATUS,
112+
FUND,
113+
FUND_LONG_NAME,
114+
DEPT,
115+
DEPT_LONG_NAME,
116+
LVL1_DIV_CODE,
117+
LVL1_DIV_LONG_NAME,
118+
LVL2_GP_CODE,
119+
LVL2_GP_LONG_NAME,
120+
UNIT,
121+
UNIT_LONG_NAME,
122+
FAO_ID,
123+
SP_NUMBER,
124+
SP_NAME,
125+
SP_STATUS,
126+
'https://ecapris.austintexas.gov/index.cfm?fuseaction=fdus.fduData&fdu_ID=' || FAO_ID as ecapris_link
127+
FROM
128+
MSTR_IA_DEV.ATD_FDUS_VW
129+
""",
108130
}

0 commit comments

Comments
 (0)