library( irs990efile ) # functions for parsing efile XMLs
library( dplyr ) # data wrangling
library( purrr ) # data wrangling
library( pander ) # formatting
library( knitr ) # formatting
library( DT ) # table printing
library( httr ) # web requests
library( xmltools ) # xml utilities
library( xml2 ) # xml utilities
library( XML ) # xml utilities
library( data.tree ) # network visualization
library( networkD3 ) # network visualization
library( igraph ) # network visualization
source( "R/rdb-functions-v2.R")
source( "R/utils.R")head( index )The concordance is an RDB to XML Crosswalk that maps all xpaths onto a specified set of variables. It is located in the irs990efile package.
The full data dictionary is available on GitHub.
concordance %>%
filter( rdb_table == table.name )
| xpath | id |
|---|---|
| /Return/ReturnData/IRS990/Form990PartVIISectionA/AverageHoursPerWeek | 1 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/AverageHoursPerWeekRelated | 2 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/Former | 3 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/HighestCompensatedEmployee | 4 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/IndividualTrusteeOrDirector | 5 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/InstitutionalTrustee | 6 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/KeyEmployee | 7 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/Name/NameBusiness/BusinessNameLine1 | 8 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/Name/NameBusiness/BusinessNameLine2 | 9 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/Name/NamePerson | 10 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/NameBusiness/BusinessNameLine1 | 11 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/NameBusiness/BusinessNameLine2 | 12 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/NamePerson | 13 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/Officer | 14 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/OtherCompensation | 15 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/ReportableCompFromOrganization | 16 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/ReportableCompFromRelatedOrgs | 17 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/Title | 18 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/AverageHoursPerWeekRltdOrgRt | 19 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/AverageHoursPerWeekRt | 20 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/BusinessName/BusinessNameLine1 | 21 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/BusinessName/BusinessNameLine1Txt | 22 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/BusinessName/BusinessNameLine2 | 23 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/BusinessName/BusinessNameLine2Txt | 24 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/FormerOfcrDirectorTrusteeInd | 25 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/HighestCompensatedEmployeeInd | 26 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/IndividualTrusteeOrDirectorInd | 27 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/InstitutionalTrusteeInd | 28 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/KeyEmployeeInd | 29 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/OfficerInd | 30 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/OtherCompensationAmt | 31 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/PersonNm | 32 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/ReportableCompFromOrgAmt | 33 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/ReportableCompFromRltdOrgAmt | 34 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/TitleTxt | 35 |
| /Return/ReturnData/IRS990/FrmrOfcrDirTrstOrKeyEmployee/BusinessName/BusinessNameLine1 | 36 |
| /Return/ReturnData/IRS990/FrmrOfcrDirTrstOrKeyEmployee/BusinessName/BusinessNameLine2 | 37 |
| /Return/ReturnData/IRS990/OfcrDirTrusteesOrKeyEmployee/BusinessName/BusinessNameLine1 | 38 |
| /Return/ReturnData/IRS990/OfcrDirTrusteesOrKeyEmployee/BusinessName/BusinessNameLine2 | 39 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionA/AverageHoursPerWeek | 40 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionA/NamePerson | 41 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionA/OtherCompensation | 42 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionA/ReportableCompFromOrganization | 43 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionA/ReportableCompFromRelatedOrgs | 44 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionA/Title | 45 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionAGrp/AverageHoursPerWeekRt | 46 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionAGrp/OtherCompensationAmt | 47 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionAGrp/PersonNm | 48 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionAGrp/ReportableCompFromOrgAmt | 49 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionAGrp/ReportableCompFromRltdOrgAmt | 50 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionAGrp/TitleTxt | 51 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/AverageHrsPerWkDevotedToPosRt | 52 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/BusinessName/BusinessNameLine1 | 53 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/BusinessName/BusinessNameLine1Txt | 54 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/BusinessName/BusinessNameLine2 | 55 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/BusinessName/BusinessNameLine2Txt | 56 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/CompensationAmt | 57 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/EmployeeBenefitProgramAmt | 58 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/PersonNm | 59 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/TitleTxt | 60 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/AvgHoursPerWkDevotedToPosition | 61 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/BusinessName/BusinessNameLine1 | 62 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/BusinessName/BusinessNameLine2 | 63 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/Compensation | 64 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/ContriToEmplBenefitPlansEtc | 65 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/Name | 66 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/PersonName | 67 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/Title | 68 |
xpaths <- as.character( t.xpaths$xpath )
el <- create_edgelist_v1( xpaths )
nd <- FromDataFrameNetwork( network=el )
print( nd )## levelName
## 1 Return
## 2 °--ReturnData
## 3 ¦--IRS990
## 4 ¦ ¦--Form990PartVIISectionA
## 5 ¦ ¦ ¦--AverageHoursPerWeek
## 6 ¦ ¦ ¦--AverageHoursPerWeekRelated
## 7 ¦ ¦ ¦--Former
## 8 ¦ ¦ ¦--HighestCompensatedEmployee
## 9 ¦ ¦ ¦--IndividualTrusteeOrDirector
## 10 ¦ ¦ ¦--InstitutionalTrustee
## 11 ¦ ¦ ¦--KeyEmployee
## 12 ¦ ¦ ¦--Name
## 13 ¦ ¦ ¦ ¦--NameBusiness
## 14 ¦ ¦ ¦ ¦ ¦--BusinessNameLine1
## 15 ¦ ¦ ¦ ¦ °--BusinessNameLine2
## 16 ¦ ¦ ¦ °--NamePerson
## 17 ¦ ¦ ¦--NameBusiness
## 18 ¦ ¦ ¦ ¦--BusinessNameLine1
## 19 ¦ ¦ ¦ °--BusinessNameLine2
## 20 ¦ ¦ ¦--NamePerson
## 21 ¦ ¦ ¦--Officer
## 22 ¦ ¦ ¦--OtherCompensation
## 23 ¦ ¦ ¦--ReportableCompFromOrganization
## 24 ¦ ¦ ¦--ReportableCompFromRelatedOrgs
## 25 ¦ ¦ °--Title
## 26 ¦ ¦--Form990PartVIISectionAGrp
## 27 ¦ ¦ ¦--AverageHoursPerWeekRltdOrgRt
## 28 ¦ ¦ ¦--AverageHoursPerWeekRt
## 29 ¦ ¦ ¦--BusinessName
## 30 ¦ ¦ ¦ ¦--BusinessNameLine1
## 31 ¦ ¦ ¦ ¦--BusinessNameLine1Txt
## 32 ¦ ¦ ¦ ¦--BusinessNameLine2
## 33 ¦ ¦ ¦ °--BusinessNameLine2Txt
## 34 ¦ ¦ ¦--FormerOfcrDirectorTrusteeInd
## 35 ¦ ¦ ¦--HighestCompensatedEmployeeInd
## 36 ¦ ¦ ¦--IndividualTrusteeOrDirectorInd
## 37 ¦ ¦ ¦--InstitutionalTrusteeInd
## 38 ¦ ¦ ¦--KeyEmployeeInd
## 39 ¦ ¦ ¦--OfficerInd
## 40 ¦ ¦ ¦--OtherCompensationAmt
## 41 ¦ ¦ ¦--PersonNm
## 42 ¦ ¦ ¦--ReportableCompFromOrgAmt
## 43 ¦ ¦ ¦--ReportableCompFromRltdOrgAmt
## 44 ¦ ¦ °--TitleTxt
## 45 ¦ ¦--FrmrOfcrDirTrstOrKeyEmployee
## 46 ¦ ¦ °--BusinessName
## 47 ¦ ¦ ¦--BusinessNameLine1
## 48 ¦ ¦ ¦--BusinessNameLine1Txt
## 49 ¦ ¦ ¦--BusinessNameLine2
## 50 ¦ ¦ °--BusinessNameLine2Txt
## 51 ¦ °--OfcrDirTrusteesOrKeyEmployee
## 52 ¦ °--BusinessName
## 53 ¦ ¦--BusinessNameLine1
## 54 ¦ ¦--BusinessNameLine1Txt
## 55 ¦ ¦--BusinessNameLine2
## 56 ¦ °--BusinessNameLine2Txt
## 57 °--IRS990EZ
## 58 ¦--Form990PartVIISectionA
## 59 ¦ ¦--AverageHoursPerWeek
## 60 ¦ ¦--AverageHoursPerWeekRelated
## 61 ¦ ¦--Former
## 62 ¦ ¦--HighestCompensatedEmployee
## 63 ¦ ¦--IndividualTrusteeOrDirector
## 64 ¦ ¦--InstitutionalTrustee
## 65 ¦ ¦--KeyEmployee
## 66 ¦ ¦--Name
## 67 ¦ ¦ ¦--NameBusiness
## 68 ¦ ¦ ¦ ¦--BusinessNameLine1
## 69 ¦ ¦ ¦ °--BusinessNameLine2
## 70 ¦ ¦ °--NamePerson
## 71 ¦ ¦--NameBusiness
## 72 ¦ ¦ ¦--BusinessNameLine1
## 73 ¦ ¦ °--BusinessNameLine2
## 74 ¦ ¦--NamePerson
## 75 ¦ ¦--Officer
## 76 ¦ ¦--OtherCompensation
## 77 ¦ ¦--ReportableCompFromOrganization
## 78 ¦ ¦--ReportableCompFromRelatedOrgs
## 79 ¦ °--Title
## 80 ¦--Form990PartVIISectionAGrp
## 81 ¦ ¦--AverageHoursPerWeekRltdOrgRt
## 82 ¦ ¦--AverageHoursPerWeekRt
## 83 ¦ ¦--BusinessName
## 84 ¦ ¦ ¦--BusinessNameLine1
## 85 ¦ ¦ ¦--BusinessNameLine1Txt
## 86 ¦ ¦ ¦--BusinessNameLine2
## 87 ¦ ¦ °--BusinessNameLine2Txt
## 88 ¦ ¦--FormerOfcrDirectorTrusteeInd
## 89 ¦ ¦--HighestCompensatedEmployeeInd
## 90 ¦ ¦--IndividualTrusteeOrDirectorInd
## 91 ¦ ¦--InstitutionalTrusteeInd
## 92 ¦ ¦--KeyEmployeeInd
## 93 ¦ ¦--OfficerInd
## 94 ¦ ¦--OtherCompensationAmt
## 95 ¦ ¦--PersonNm
## 96 ¦ ¦--ReportableCompFromOrgAmt
## 97 ¦ ¦--ReportableCompFromRltdOrgAmt
## 98 ¦ °--TitleTxt
## 99 ¦--OfficerDirectorTrusteeEmplGrp
## 100 ¦ °--... 6 nodes w/ 4 sub
## 101 °--... 1 nodes w/ 25 subSetGraphStyle( nd, rankdir = "LR")
SetEdgeStyle( nd, arrowhead = "vee", color = "grey20", penwidth = 2 )
SetNodeStyle( nd,
style = "filled,rounded",
shape = "box",
fillcolor = "LightBlue",
fontname = "helvetica",
fontcolor="black",
tooltip = GetDefaultTooltip )
# SetNodeStyle(acme$IT, fillcolor = "LightBlue", penwidth = "5px")
plot( nd )# old version
find_group_names( table.name )## [1] "//Form990PartVIISectionA" "//Form990PartVIISectionAGrp"
## [3] "//FrmrOfcrDirTrstOrKeyEmployee" "//OfcrDirTrusteesOrKeyEmployee"
## [5] "//OfficerDirectorTrusteeEmplGrp" "//OfficerDirectorTrusteeKeyEmpl"# new version
find_table_headers( table.name )## [1] "//Form990PartVIISectionA/AverageHoursPerWeek"
## [2] "//Form990PartVIISectionA/AverageHoursPerWeekRelated"
## [3] "//Form990PartVIISectionA/CompensationFromOtherSources"
## [4] "//Form990PartVIISectionA/Former"
## [5] "//Form990PartVIISectionA/FormersListedIn1A"
## [6] "//Form990PartVIISectionA/HighestCompensatedEmployee"
## [7] "//Form990PartVIISectionA/IndividualTrusteeOrDirector"
## [8] "//Form990PartVIISectionA/InstitutionalTrustee"
## [9] "//Form990PartVIISectionA/KeyEmployee"
## [10] "//Form990PartVIISectionA/Line1ATotalGT150K"
## [11] "//Form990PartVIISectionA/Name"
## [12] "//Form990PartVIISectionA/NameBusiness"
## [13] "//Form990PartVIISectionA/NamePerson"
## [14] "//Form990PartVIISectionA/NoListedPersonsCompensated"
## [15] "//Form990PartVIISectionA/NumberIndividualsGT100K"
## [16] "//Form990PartVIISectionA/Officer"
## [17] "//Form990PartVIISectionA/OtherCompensation"
## [18] "//Form990PartVIISectionA/ReportableCompFromOrganization"
## [19] "//Form990PartVIISectionA/ReportableCompFromRelatedOrgs"
## [20] "//Form990PartVIISectionA/Title"
## [21] "//Form990PartVIISectionA/TotalColumnD"
## [22] "//Form990PartVIISectionA/TotalColumnE"
## [23] "//Form990PartVIISectionA/TotalColumnF"
## [24] "//Form990PartVIISectionAGrp/AverageHoursPerWeekRltdOrgRt"
## [25] "//Form990PartVIISectionAGrp/AverageHoursPerWeekRt"
## [26] "//Form990PartVIISectionAGrp/BusinessName"
## [27] "//Form990PartVIISectionAGrp/FormerOfcrDirectorTrusteeInd"
## [28] "//Form990PartVIISectionAGrp/HighestCompensatedEmployeeInd"
## [29] "//Form990PartVIISectionAGrp/IndividualTrusteeOrDirectorInd"
## [30] "//Form990PartVIISectionAGrp/InstitutionalTrusteeInd"
## [31] "//Form990PartVIISectionAGrp/KeyEmployeeInd"
## [32] "//Form990PartVIISectionAGrp/OfficerInd"
## [33] "//Form990PartVIISectionAGrp/OtherCompensationAmt"
## [34] "//Form990PartVIISectionAGrp/PersonNm"
## [35] "//Form990PartVIISectionAGrp/ReportableCompFromOrgAmt"
## [36] "//Form990PartVIISectionAGrp/ReportableCompFromRltdOrgAmt"
## [37] "//Form990PartVIISectionAGrp/TitleTxt"
## [38] "//FrmrOfcrDirTrstOrKeyEmployee/BusinessNameLine1"
## [39] "//FrmrOfcrDirTrstOrKeyEmployee/BusinessNameLine2"
## [40] "//OfcrDirTrusteesOrKeyEmployee/BusinessNameLine1"
## [41] "//OfcrDirTrusteesOrKeyEmployee/BusinessNameLine2"
## [42] "//OfficerDirectorTrusteeEmplGrp/AverageHrsPerWkDevotedToPosRt"
## [43] "//OfficerDirectorTrusteeEmplGrp/BusinessName"
## [44] "//OfficerDirectorTrusteeEmplGrp/CompensationAmt"
## [45] "//OfficerDirectorTrusteeEmplGrp/EmployeeBenefitProgramAmt"
## [46] "//OfficerDirectorTrusteeEmplGrp/PersonNm"
## [47] "//OfficerDirectorTrusteeEmplGrp/TitleTxt"
## [48] "//OfficerDirectorTrusteeKeyEmpl/AddressForeign"
## [49] "//OfficerDirectorTrusteeKeyEmpl/AddressUS"
## [50] "//OfficerDirectorTrusteeKeyEmpl/AvgHoursPerWkDevotedToPosition"
## [51] "//OfficerDirectorTrusteeKeyEmpl/BusinessName"
## [52] "//OfficerDirectorTrusteeKeyEmpl/Compensation"
## [53] "//OfficerDirectorTrusteeKeyEmpl/ContriToEmplBenefitPlansEtc"
## [54] "//OfficerDirectorTrusteeKeyEmpl/Name"
## [55] "//OfficerDirectorTrusteeKeyEmpl/PersonName"
## [56] "//OfficerDirectorTrusteeKeyEmpl/Title"# manual
if( ! is.null(table.headers) )
{table.headers}
# create new folder
dir.create( folder.name )
setwd( folder.name )
# create sample
set.seed( 1234 )
sample.index <- sample_n( index, sample.size )
write.csv( index, "sample-index.csv", row.names=F )
sample.urls <- sample.index$URL
# erase existing log files
file.create("XPATH-LOG.txt")
file.create("FAIL-LOG.txt")
start.build.time <- Sys.time() # --------------------
results.list <- list()
xpath.list <- list()
for( i in 1:length( sample.urls ) )
{
url <- sample.urls[i]
results.list[[i]] <-
build_rdb_table_v2( url,
table.name,
table.headers=table.headers )
xpath.list[[i]] <-
get_table_xpaths( url,
table.name,
table.headers=table.headers )
if( i %% 100 == 0 ){ print(i) }
}
end.build.time <- Sys.time() # --------------------
df <- dplyr::bind_rows( results.list )
fail.log <- readLines( "FAIL-LOG.txt" )
all.xpaths <- xpath.list %>% unlist() %>% unique() %>% sort()
cat( all.xpaths, sep="\n", file="XPATH-LOG.txt" )TABLE: F9-P07-T01-COMPENSATION
## [1] "NUMBER OF RETURNS PROCESSED: 1000"
## [1] "TOTAL NUMBER OF TABLE ROWS: 10108"TOTAL RUN TIME:
## Time difference of 47.10262 mins
## Time difference of -0.7850436 hourswrite.csv( df, paste0( folder.name, "/", table.name, ".csv" ), row.names=F )
head(df) Are all names standardized?
names(df)## [1] "OBJECT_ID" "EIN"
## [3] "NAME" "TAXYR"
## [5] "FORMTYPE" "URL"
## [7] "F9_07_COMP_DTK_NAME_PERS" "F9_07_COMP_DTK_TITLE"
## [9] "F9_07_COMP_DTK_AVE_HOUR_WEEK" "F9_07_COMP_DTK_AVE_HOUR_WEEK_RL"
## [11] "F9_07_COMP_DTK_POS_INDIV_TRUST_X" "F9_07_COMP_DTK_COMP_ORG"
## [13] "F9_07_COMP_DTK_COMP_RLTD" "F9_07_COMP_DTK_COMP_OTH"
## [15] "F9_07_COMP_DTK_POS_INST_TRUST_X" "F9_07_COMP_DTK_EMPL_BEN"
## [17] "ExpenseAccountOtherAllwncAmt" "F9_07_COMP_DTK_POS_OFF_X"
## [19] "AddressLine1" "City"
## [21] "State" "ZIPCode"
## [23] "ExpenseAccountOtherAllowances" "F9_07_COMP_DTK_POS_KEY_EMPL_X"
## [25] "F9_07_COMP_DTK_POS_HIGH_COMP_X" "F9_07_COMP_DTK_NAME_ORG_L1"
## [27] "F9_07_COMP_DTK_POS_FORMER_X" "AddressLine2"
Check for xpaths that were used in the tables but do not exist in the Concordance file.
c.table <- dplyr::filter( concordance, rdb_table == table.name )
xpath.concordance <- c.table$xpath
setdiff( all.xpaths, xpath.concordance )## [1] "/Return/ReturnData/IRS990/Form990PartVIISectionAGrp/BusinessName"
## [2] "/Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/BusinessName"
## [3] "/Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/AddressUS"
## [4] "/Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/AddressUS/AddressLine1"
## [5] "/Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/AddressUS/AddressLine2"
## [6] "/Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/AddressUS/City"
## [7] "/Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/AddressUS/State"
## [8] "/Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/AddressUS/ZIPCode"
## [9] "/Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/BusinessName"Capture to update the Concordance file.
these <- setdiff( all.xpaths, xpath.concordance )
cat( these, sep="\n", file=paste0( folder.name, "/MISSING-XPATHS.txt" ) )Check the status of failed URLs.
if( length(fail.log) > 0 )
{
sapply( fail.log, get_message ) %>%
table() %>%
pander()
} else {
print("ALL URLs ARE VALID")
}| Client error: (404) Not Found |
|---|
| 1 |
The create_edgelist() functions v1 and v2 are in R/utils.R.
V1 retains the root nodes Return and ReturnData.
V2 drops the root nodes and uses form versions “990” and “990EZ” as the root to visualize the sets of tables separately.
# KEEP ROOT NOTES Return & ReturnData
el <- create_edgelist_v1( all.xpaths )Edgelist format:
| V1 | V2 |
|---|---|
| Return | ReturnData |
| ReturnData | IRS990 |
| IRS990 | Form990PartVIISectionAGrp |
| Form990PartVIISectionAGrp | PersonNm |
| Form990PartVIISectionAGrp | TitleTxt |
| Form990PartVIISectionAGrp | AverageHoursPerWeekRt |
| Form990PartVIISectionAGrp | IndividualTrusteeOrDirectorInd |
| Form990PartVIISectionAGrp | OfficerInd |
| Form990PartVIISectionAGrp | ReportableCompFromOrgAmt |
| Form990PartVIISectionAGrp | ReportableCompFromRltdOrgAmt |
# library( data.tree )
nd <- FromDataFrameNetwork( network=el )
print( nd )## levelName
## 1 Return
## 2 °--ReturnData
## 3 ¦--IRS990
## 4 ¦ ¦--Form990PartVIISectionA
## 5 ¦ ¦ ¦--AverageHoursPerWeek
## 6 ¦ ¦ ¦--AverageHoursPerWeekRelated
## 7 ¦ ¦ ¦--Former
## 8 ¦ ¦ ¦--HighestCompensatedEmployee
## 9 ¦ ¦ ¦--IndividualTrusteeOrDirector
## 10 ¦ ¦ ¦--InstitutionalTrustee
## 11 ¦ ¦ ¦--KeyEmployee
## 12 ¦ ¦ ¦--NamePerson
## 13 ¦ ¦ ¦--Officer
## 14 ¦ ¦ ¦--OtherCompensation
## 15 ¦ ¦ ¦--ReportableCompFromOrganization
## 16 ¦ ¦ ¦--ReportableCompFromRelatedOrgs
## 17 ¦ ¦ °--Title
## 18 ¦ °--Form990PartVIISectionAGrp
## 19 ¦ ¦--AverageHoursPerWeekRltdOrgRt
## 20 ¦ ¦--AverageHoursPerWeekRt
## 21 ¦ ¦--BusinessName
## 22 ¦ ¦ ¦--BusinessNameLine1
## 23 ¦ ¦ °--BusinessNameLine1Txt
## 24 ¦ ¦--FormerOfcrDirectorTrusteeInd
## 25 ¦ ¦--HighestCompensatedEmployeeInd
## 26 ¦ ¦--IndividualTrusteeOrDirectorInd
## 27 ¦ ¦--InstitutionalTrusteeInd
## 28 ¦ ¦--KeyEmployeeInd
## 29 ¦ ¦--OfficerInd
## 30 ¦ ¦--OtherCompensationAmt
## 31 ¦ ¦--PersonNm
## 32 ¦ ¦--ReportableCompFromOrgAmt
## 33 ¦ ¦--ReportableCompFromRltdOrgAmt
## 34 ¦ °--TitleTxt
## 35 °--IRS990EZ
## 36 ¦--OfficerDirectorTrusteeEmplGrp
## 37 ¦ ¦--AverageHrsPerWkDevotedToPosRt
## 38 ¦ ¦--BusinessName
## 39 ¦ ¦ ¦--BusinessNameLine1
## 40 ¦ ¦ °--BusinessNameLine1Txt
## 41 ¦ ¦--CompensationAmt
## 42 ¦ ¦--EmployeeBenefitProgramAmt
## 43 ¦ ¦--PersonNm
## 44 ¦ °--TitleTxt
## 45 °--OfficerDirectorTrusteeKeyEmpl
## 46 ¦--AddressUS
## 47 ¦ ¦--AddressLine1
## 48 ¦ ¦--AddressLine2
## 49 ¦ ¦--City
## 50 ¦ ¦--State
## 51 ¦ °--ZIPCode
## 52 ¦--AvgHoursPerWkDevotedToPosition
## 53 ¦--BusinessName
## 54 ¦ ¦--BusinessNameLine1
## 55 ¦ °--BusinessNameLine1Txt
## 56 ¦--Compensation
## 57 ¦--ContriToEmplBenefitPlansEtc
## 58 ¦--PersonName
## 59 °--TitleSetGraphStyle( nd, rankdir = "LR")
SetEdgeStyle( nd, arrowhead = "vee", color = "grey20", penwidth = 2 )
SetNodeStyle( nd,
style = "filled,rounded",
shape = "box",
fillcolor = "LightBlue",
fontname = "helvetica",
fontcolor="black",
tooltip = GetDefaultTooltip )
# SetNodeStyle(acme$IT, fillcolor = "LightBlue", penwidth = "5px")
plot( nd )# library( networkD3 )
# net <- ToDataFrameNetwork( nd, "name" )
# simpleNetwork( net, fontSize = 12 )
simpleNetwork( el, fontSize = 12 )#plot with networkD3
radnet <- ToListExplicit( nd, unname = TRUE )
radialNetwork( radnet, fontSize = 12 )
t.xpaths <-
concordance %>%
filter( rdb_table == table.name ) %>%
select( xpath ) %>%
unique() %>%
arrange( xpath ) %>%
mutate( id=row_number() )
t.xpaths %>%
kable( align="l" )| xpath | id |
|---|---|
| /Return/ReturnData/IRS990/Form990PartVIISectionA/AverageHoursPerWeek | 1 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/AverageHoursPerWeekRelated | 2 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/Former | 3 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/HighestCompensatedEmployee | 4 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/IndividualTrusteeOrDirector | 5 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/InstitutionalTrustee | 6 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/KeyEmployee | 7 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/Name/NameBusiness/BusinessNameLine1 | 8 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/Name/NameBusiness/BusinessNameLine2 | 9 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/Name/NamePerson | 10 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/NameBusiness/BusinessNameLine1 | 11 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/NameBusiness/BusinessNameLine2 | 12 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/NamePerson | 13 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/Officer | 14 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/OtherCompensation | 15 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/ReportableCompFromOrganization | 16 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/ReportableCompFromRelatedOrgs | 17 |
| /Return/ReturnData/IRS990/Form990PartVIISectionA/Title | 18 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/AverageHoursPerWeekRltdOrgRt | 19 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/AverageHoursPerWeekRt | 20 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/BusinessName/BusinessNameLine1 | 21 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/BusinessName/BusinessNameLine1Txt | 22 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/BusinessName/BusinessNameLine2 | 23 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/BusinessName/BusinessNameLine2Txt | 24 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/FormerOfcrDirectorTrusteeInd | 25 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/HighestCompensatedEmployeeInd | 26 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/IndividualTrusteeOrDirectorInd | 27 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/InstitutionalTrusteeInd | 28 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/KeyEmployeeInd | 29 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/OfficerInd | 30 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/OtherCompensationAmt | 31 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/PersonNm | 32 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/ReportableCompFromOrgAmt | 33 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/ReportableCompFromRltdOrgAmt | 34 |
| /Return/ReturnData/IRS990/Form990PartVIISectionAGrp/TitleTxt | 35 |
| /Return/ReturnData/IRS990/FrmrOfcrDirTrstOrKeyEmployee/BusinessName/BusinessNameLine1 | 36 |
| /Return/ReturnData/IRS990/FrmrOfcrDirTrstOrKeyEmployee/BusinessName/BusinessNameLine2 | 37 |
| /Return/ReturnData/IRS990/OfcrDirTrusteesOrKeyEmployee/BusinessName/BusinessNameLine1 | 38 |
| /Return/ReturnData/IRS990/OfcrDirTrusteesOrKeyEmployee/BusinessName/BusinessNameLine2 | 39 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionA/AverageHoursPerWeek | 40 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionA/NamePerson | 41 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionA/OtherCompensation | 42 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionA/ReportableCompFromOrganization | 43 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionA/ReportableCompFromRelatedOrgs | 44 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionA/Title | 45 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionAGrp/AverageHoursPerWeekRt | 46 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionAGrp/OtherCompensationAmt | 47 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionAGrp/PersonNm | 48 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionAGrp/ReportableCompFromOrgAmt | 49 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionAGrp/ReportableCompFromRltdOrgAmt | 50 |
| /Return/ReturnData/IRS990EZ/Form990PartVIISectionAGrp/TitleTxt | 51 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/AverageHrsPerWkDevotedToPosRt | 52 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/BusinessName/BusinessNameLine1 | 53 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/BusinessName/BusinessNameLine1Txt | 54 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/BusinessName/BusinessNameLine2 | 55 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/BusinessName/BusinessNameLine2Txt | 56 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/CompensationAmt | 57 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/EmployeeBenefitProgramAmt | 58 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/PersonNm | 59 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeEmplGrp/TitleTxt | 60 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/AvgHoursPerWkDevotedToPosition | 61 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/BusinessName/BusinessNameLine1 | 62 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/BusinessName/BusinessNameLine2 | 63 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/Compensation | 64 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/ContriToEmplBenefitPlansEtc | 65 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/Name | 66 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/PersonName | 67 |
| /Return/ReturnData/IRS990EZ/OfficerDirectorTrusteeKeyEmpl/Title | 68 |
Identify all unique groups used in xpaths.
if( is.null(table.headers) )
{ find_table_headers( table.name ) } else
table.headers## [1] "//Form990PartVIISectionA/AverageHoursPerWeek"
## [2] "//Form990PartVIISectionA/AverageHoursPerWeekRelated"
## [3] "//Form990PartVIISectionA/CompensationFromOtherSources"
## [4] "//Form990PartVIISectionA/Former"
## [5] "//Form990PartVIISectionA/FormersListedIn1A"
## [6] "//Form990PartVIISectionA/HighestCompensatedEmployee"
## [7] "//Form990PartVIISectionA/IndividualTrusteeOrDirector"
## [8] "//Form990PartVIISectionA/InstitutionalTrustee"
## [9] "//Form990PartVIISectionA/KeyEmployee"
## [10] "//Form990PartVIISectionA/Line1ATotalGT150K"
## [11] "//Form990PartVIISectionA/Name"
## [12] "//Form990PartVIISectionA/NameBusiness"
## [13] "//Form990PartVIISectionA/NamePerson"
## [14] "//Form990PartVIISectionA/NoListedPersonsCompensated"
## [15] "//Form990PartVIISectionA/NumberIndividualsGT100K"
## [16] "//Form990PartVIISectionA/Officer"
## [17] "//Form990PartVIISectionA/OtherCompensation"
## [18] "//Form990PartVIISectionA/ReportableCompFromOrganization"
## [19] "//Form990PartVIISectionA/ReportableCompFromRelatedOrgs"
## [20] "//Form990PartVIISectionA/Title"
## [21] "//Form990PartVIISectionA/TotalColumnD"
## [22] "//Form990PartVIISectionA/TotalColumnE"
## [23] "//Form990PartVIISectionA/TotalColumnF"
## [24] "//Form990PartVIISectionAGrp/AverageHoursPerWeekRltdOrgRt"
## [25] "//Form990PartVIISectionAGrp/AverageHoursPerWeekRt"
## [26] "//Form990PartVIISectionAGrp/BusinessName"
## [27] "//Form990PartVIISectionAGrp/FormerOfcrDirectorTrusteeInd"
## [28] "//Form990PartVIISectionAGrp/HighestCompensatedEmployeeInd"
## [29] "//Form990PartVIISectionAGrp/IndividualTrusteeOrDirectorInd"
## [30] "//Form990PartVIISectionAGrp/InstitutionalTrusteeInd"
## [31] "//Form990PartVIISectionAGrp/KeyEmployeeInd"
## [32] "//Form990PartVIISectionAGrp/OfficerInd"
## [33] "//Form990PartVIISectionAGrp/OtherCompensationAmt"
## [34] "//Form990PartVIISectionAGrp/PersonNm"
## [35] "//Form990PartVIISectionAGrp/ReportableCompFromOrgAmt"
## [36] "//Form990PartVIISectionAGrp/ReportableCompFromRltdOrgAmt"
## [37] "//Form990PartVIISectionAGrp/TitleTxt"
## [38] "//FrmrOfcrDirTrstOrKeyEmployee/BusinessNameLine1"
## [39] "//FrmrOfcrDirTrstOrKeyEmployee/BusinessNameLine2"
## [40] "//OfcrDirTrusteesOrKeyEmployee/BusinessNameLine1"
## [41] "//OfcrDirTrusteesOrKeyEmployee/BusinessNameLine2"
## [42] "//OfficerDirectorTrusteeEmplGrp/AverageHrsPerWkDevotedToPosRt"
## [43] "//OfficerDirectorTrusteeEmplGrp/BusinessName"
## [44] "//OfficerDirectorTrusteeEmplGrp/CompensationAmt"
## [45] "//OfficerDirectorTrusteeEmplGrp/EmployeeBenefitProgramAmt"
## [46] "//OfficerDirectorTrusteeEmplGrp/PersonNm"
## [47] "//OfficerDirectorTrusteeEmplGrp/TitleTxt"
## [48] "//OfficerDirectorTrusteeKeyEmpl/AddressForeign"
## [49] "//OfficerDirectorTrusteeKeyEmpl/AddressUS"
## [50] "//OfficerDirectorTrusteeKeyEmpl/AvgHoursPerWkDevotedToPosition"
## [51] "//OfficerDirectorTrusteeKeyEmpl/BusinessName"
## [52] "//OfficerDirectorTrusteeKeyEmpl/Compensation"
## [53] "//OfficerDirectorTrusteeKeyEmpl/ContriToEmplBenefitPlansEtc"
## [54] "//OfficerDirectorTrusteeKeyEmpl/Name"
## [55] "//OfficerDirectorTrusteeKeyEmpl/PersonName"
## [56] "//OfficerDirectorTrusteeKeyEmpl/Title"Groups can then be used to extract the 1:M RDB tables:
if( is.null(table.headers) )
{ table.headers <- find_table_headers( table.name ) }
demo.url <- "https://nccs-efile.s3.us-east-1.amazonaws.com/xml/201913079349300206_public.xml"
doc <- xml2::read_xml( demo.url )
xml2::xml_ns_strip( doc )
df.demo <- get_table_v2( doc, table.name, table.headers )
df.demo ## # A tibble: 5 × 9
## PersonNm Title…¹ Avera…² Avera…³ Indiv…⁴ Repor…⁵ Repor…⁶ Other…⁷ Insti…⁸
## <chr> <chr> <chr> <chr> <chr> <chr> <chr> <chr> <chr>
## 1 Martin … Exec D… 30 0 X 13500 0 0 <NA>
## 2 Scott C… Presid… 10 0 <NA> 0 0 0 X
## 3 Rick Sa… Vice P… 1 0 <NA> 0 0 0 X
## 4 Melinda… Secret… 3 0 <NA> 0 0 0 X
## 5 Chris O… Treasu… 3 0 <NA> 0 0 0 X
## # … with abbreviated variable names ¹TitleTxt, ²AverageHoursPerWeekRt,
## # ³AverageHoursPerWeekRltdOrgRt, ⁴IndividualTrusteeOrDirectorInd,
## # ⁵ReportableCompFromOrgAmt, ⁶ReportableCompFromRltdOrgAmt,
## # ⁷OtherCompensationAmt, ⁸InstitutionalTrusteeIndMapping of xml variable names to concordance variable names:
get_var_map( table.name=table.name )A table with proper names:
v.map <- get_var_map( table.name=table.name )
if( ! is.null(df.demo) )
{re_name( df.demo, v.map ) %>% as.data.frame()}## F9_07_COMP_DTK_NAME_PERS F9_07_COMP_DTK_TITLE
## 1 Martin Abraham Exec Director
## 2 Scott Collins President
## 3 Rick Sartori Vice President
## 4 Melinda Ulle Secretary
## 5 Chris Obert Treasurer
## F9_07_COMP_DTK_AVE_HOUR_WEEK F9_07_COMP_DTK_AVE_HOUR_WEEK_RL
## 1 30 0
## 2 10 0
## 3 1 0
## 4 3 0
## 5 3 0
## F9_07_COMP_DTK_POS_INDIV_TRUST_X F9_07_COMP_DTK_COMP_ORG
## 1 X 13500
## 2 <NA> 0
## 3 <NA> 0
## 4 <NA> 0
## 5 <NA> 0
## F9_07_COMP_DTK_COMP_RLTD F9_07_COMP_DTK_COMP_OTH
## 1 0 0
## 2 0 0
## 3 0 0
## 4 0 0
## 5 0 0
## F9_07_COMP_DTK_POS_INST_TRUST_X
## 1 <NA>
## 2 X
## 3 X
## 4 X
## 5 X
Demo of functions - demo data only.
[[1]]
<Form990PartVIISectionAGrp>
<PersonNm>
{text}
<TitleTxt>
{text}
<AverageHoursPerWeekRt>
{text}
<IndividualTrusteeOrDirectorInd>
{text}
<OfficerInd>
{text}
<ReportableCompFromOrgAmt>
{text}
<ReportableCompFromRltdOrgAmt>
{text}
<OtherCompensationAmt>
{text}node.list[[1]] %>% xml_parent() %>% xmltools::xml_view_trees() (1) ------------
+-- Form990PartVIISectionAGrp
+-- PersonNm
+-- TitleTxt
+-- AverageHoursPerWeekRt
+-- IndividualTrusteeOrDirectorInd
+-- OfficerInd
+-- ReportableCompFromOrgAmt
+-- ReportableCompFromRltdOrgAmt
++-- OtherCompensationAmt
(2) ------------
+-- Form990PartVIISectionAGrp
+-- PersonNm
+-- TitleTxt
+-- AverageHoursPerWeekRt
+-- IndividualTrusteeOrDirectorInd
+-- OfficerInd
+-- ReportableCompFromOrgAmt
+-- ReportableCompFromRltdOrgAmt
++-- OtherCompensationAmt