library( irs990efile ) # functions for parsing efile XMLs
library( dplyr ) # data wrangling
library( purrr ) # data wrangling
library( pander ) # formatting
library( knitr ) # formatting
library( DT ) # table printing
library( httr ) # web requests
library( xmltools ) # xml utilities
library( xml2 ) # xml utilities
library( XML ) # xml utilities
library( data.tree ) # network visualization
library( networkD3 ) # network visualization
library( igraph ) # network visualization
source( "R/rdb-functions-v2.R")
source( "R/utils.R")head( index )The concordance is an RDB to XML Crosswalk that maps all xpaths onto a specified set of variables. It is located in the irs990efile package.
The full data dictionary is available on GitHub.
concordance %>%
filter( rdb_table == table.name )
| xpath | id |
|---|---|
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressForeign/AddressLine1 | 1 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressForeign/AddressLine2 | 2 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressForeign/City | 3 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressForeign/Country | 4 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressForeign/PostalCode | 5 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressForeign/ProvinceOrState | 6 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressUS/AddressLine1 | 7 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressUS/AddressLine2 | 8 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressUS/City | 9 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressUS/State | 10 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressUS/ZIPCode | 11 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/CodeVAmount | 12 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/DirectControllingEntity/NameBusiness/BusinessNameLine1 | 13 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/DirectControllingEntity/NameBusiness/BusinessNameLine2 | 14 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/DirectControllingEntity/NotApplicable | 15 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/DirectControllingEntityNA | 16 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/DirectControllingEntityName/BusinessNameLine1 | 17 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/DirectControllingEntityName/BusinessNameLine2 | 18 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/DisproportionateAllocations | 19 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/EIN | 20 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/GeneralOrManagingPartner | 21 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/LegalDomicile/ForeignCountry | 22 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/LegalDomicile/State | 23 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/LegalDomicileForeignCountry | 24 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/LegalDomicileState | 25 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/NameOfRelatedOrg/BusinessNameLine1 | 26 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/NameOfRelatedOrg/BusinessNameLine2 | 27 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/PercentageOwnership | 28 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/PredominantIncomeType | 29 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/PrimaryActivity | 30 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/ShareOfEOYAssets | 31 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/ShareOfTotalIncome | 32 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/DirectControllingEntityName/BusinessNameLine1 | 33 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/DirectControllingEntityName/BusinessNameLine1Txt | 34 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/DirectControllingEntityName/BusinessNameLine2 | 35 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/DirectControllingEntityName/BusinessNameLine2Txt | 36 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/DirectControllingNACd | 37 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/DisproportionateAllocationsInd | 38 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/EIN | 39 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/AddressLine1 | 40 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/AddressLine1Txt | 41 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/AddressLine2 | 42 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/AddressLine2Txt | 43 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/City | 44 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/CityNm | 45 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/Country | 46 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/CountryCd | 47 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/ForeignPostalCd | 48 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/PostalCode | 49 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/ProvinceOrState | 50 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/ProvinceOrStateNm | 51 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/GeneralOrManagingPartnerInd | 52 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/LegalDomicileForeignCountryCd | 53 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/LegalDomicileStateCd | 54 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/OwnershipPct | 55 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/PredominantIncomeTypeTxt | 56 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/PrimaryActivitiesTxt | 57 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/RelatedOrganizationName/BusinessNameLine1 | 58 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/RelatedOrganizationName/BusinessNameLine1Txt | 59 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/RelatedOrganizationName/BusinessNameLine2 | 60 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/RelatedOrganizationName/BusinessNameLine2Txt | 61 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ShareOfEOYAssetsAmt | 62 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ShareOfTotalIncomeAmt | 63 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/UBICodeVAmt | 64 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/AddressLine1 | 65 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/AddressLine1Txt | 66 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/AddressLine2 | 67 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/AddressLine2Txt | 68 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/City | 69 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/CityNm | 70 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/State | 71 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/StateAbbreviationCd | 72 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/ZIPCd | 73 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/ZIPCode | 74 |
xpaths <- as.character( t.xpaths$xpath )
el <- create_edgelist_v1( xpaths )
nd <- FromDataFrameNetwork( network=el )
print( nd )## levelName
## 1 Return
## 2 °--ReturnData
## 3 °--IRS990ScheduleR
## 4 ¦--Form990ScheduleRPartIII
## 5 ¦ ¦--AddressForeign
## 6 ¦ ¦ ¦--AddressLine1
## 7 ¦ ¦ ¦--AddressLine2
## 8 ¦ ¦ ¦--City
## 9 ¦ ¦ ¦--Country
## 10 ¦ ¦ ¦--PostalCode
## 11 ¦ ¦ °--ProvinceOrState
## 12 ¦ ¦--AddressUS
## 13 ¦ ¦ ¦--AddressLine1
## 14 ¦ ¦ ¦--AddressLine2
## 15 ¦ ¦ ¦--City
## 16 ¦ ¦ ¦--State
## 17 ¦ ¦ °--ZIPCode
## 18 ¦ ¦--CodeVAmount
## 19 ¦ ¦--DirectControllingEntity
## 20 ¦ ¦ ¦--NameBusiness
## 21 ¦ ¦ ¦ ¦--BusinessNameLine1
## 22 ¦ ¦ ¦ °--BusinessNameLine2
## 23 ¦ ¦ °--NotApplicable
## 24 ¦ ¦--DirectControllingEntityNA
## 25 ¦ ¦--DirectControllingEntityName
## 26 ¦ ¦ ¦--BusinessNameLine1
## 27 ¦ ¦ ¦--BusinessNameLine2
## 28 ¦ ¦ ¦--BusinessNameLine1Txt
## 29 ¦ ¦ °--BusinessNameLine2Txt
## 30 ¦ ¦--DisproportionateAllocations
## 31 ¦ ¦--EIN
## 32 ¦ ¦--GeneralOrManagingPartner
## 33 ¦ ¦--LegalDomicile
## 34 ¦ ¦ ¦--ForeignCountry
## 35 ¦ ¦ °--State
## 36 ¦ ¦--LegalDomicileForeignCountry
## 37 ¦ ¦--LegalDomicileState
## 38 ¦ ¦--NameOfRelatedOrg
## 39 ¦ ¦ ¦--BusinessNameLine1
## 40 ¦ ¦ °--BusinessNameLine2
## 41 ¦ ¦--PercentageOwnership
## 42 ¦ ¦--PredominantIncomeType
## 43 ¦ ¦--PrimaryActivity
## 44 ¦ ¦--ShareOfEOYAssets
## 45 ¦ °--ShareOfTotalIncome
## 46 °--IdRelatedOrgTxblPartnershipGrp
## 47 ¦--DirectControllingEntityName
## 48 ¦ ¦--BusinessNameLine1
## 49 ¦ ¦--BusinessNameLine2
## 50 ¦ ¦--BusinessNameLine1Txt
## 51 ¦ °--BusinessNameLine2Txt
## 52 ¦--DirectControllingNACd
## 53 ¦--DisproportionateAllocationsInd
## 54 ¦--EIN
## 55 ¦--ForeignAddress
## 56 ¦ ¦--AddressLine1
## 57 ¦ ¦--AddressLine1Txt
## 58 ¦ ¦--AddressLine2
## 59 ¦ ¦--AddressLine2Txt
## 60 ¦ ¦--City
## 61 ¦ ¦--CityNm
## 62 ¦ ¦--Country
## 63 ¦ ¦--CountryCd
## 64 ¦ ¦--ForeignPostalCd
## 65 ¦ ¦--PostalCode
## 66 ¦ ¦--ProvinceOrState
## 67 ¦ °--ProvinceOrStateNm
## 68 ¦--GeneralOrManagingPartnerInd
## 69 ¦--LegalDomicileForeignCountryCd
## 70 ¦--LegalDomicileStateCd
## 71 ¦--OwnershipPct
## 72 ¦--PredominantIncomeTypeTxt
## 73 ¦--PrimaryActivitiesTxt
## 74 ¦--RelatedOrganizationName
## 75 ¦ ¦--BusinessNameLine1
## 76 ¦ ¦--BusinessNameLine1Txt
## 77 ¦ ¦--BusinessNameLine2
## 78 ¦ °--BusinessNameLine2Txt
## 79 ¦--ShareOfEOYAssetsAmt
## 80 ¦--ShareOfTotalIncomeAmt
## 81 ¦--UBICodeVAmt
## 82 °--USAddress
## 83 ¦--AddressLine1
## 84 ¦--AddressLine1Txt
## 85 ¦--AddressLine2
## 86 ¦--AddressLine2Txt
## 87 ¦--City
## 88 ¦--CityNm
## 89 ¦--State
## 90 ¦--StateAbbreviationCd
## 91 ¦--ZIPCd
## 92 °--ZIPCodeSetGraphStyle( nd, rankdir = "LR")
SetEdgeStyle( nd, arrowhead = "vee", color = "grey20", penwidth = 2 )
SetNodeStyle( nd,
style = "filled,rounded",
shape = "box",
fillcolor = "LightBlue",
fontname = "helvetica",
fontcolor="black",
tooltip = GetDefaultTooltip )
# SetNodeStyle(acme$IT, fillcolor = "LightBlue", penwidth = "5px")
plot( nd )# old version
find_group_names( table.name )## [1] "//Form990ScheduleRPartIII" "//IdRelatedOrgTxblPartnershipGrp"# new version
find_table_headers( table.name )## [1] "//Form990ScheduleRPartIII/AddressForeign"
## [2] "//Form990ScheduleRPartIII/AddressUS"
## [3] "//Form990ScheduleRPartIII/EIN"
## [4] "//Form990ScheduleRPartIII/NameOfRelatedOrg"
## [5] "//Form990ScheduleRPartIII/PrimaryActivity"
## [6] "//Form990ScheduleRPartIII/LegalDomicile"
## [7] "//Form990ScheduleRPartIII/LegalDomicileForeignCountry"
## [8] "//Form990ScheduleRPartIII/LegalDomicileState"
## [9] "//Form990ScheduleRPartIII/DirectControllingEntityNA"
## [10] "//Form990ScheduleRPartIII/DirectControllingEntity"
## [11] "//Form990ScheduleRPartIII/DirectControllingEntityName"
## [12] "//Form990ScheduleRPartIII/PredominantIncomeType"
## [13] "//Form990ScheduleRPartIII/ShareOfTotalIncome"
## [14] "//Form990ScheduleRPartIII/ShareOfEOYAssets"
## [15] "//Form990ScheduleRPartIII/DisproportionateAllocations"
## [16] "//Form990ScheduleRPartIII/CodeVAmount"
## [17] "//Form990ScheduleRPartIII/GeneralOrManagingPartner"
## [18] "//Form990ScheduleRPartIII/PercentageOwnership"
## [19] "//IdRelatedOrgTxblPartnershipGrp/ForeignAddress"
## [20] "//IdRelatedOrgTxblPartnershipGrp/USAddress"
## [21] "//IdRelatedOrgTxblPartnershipGrp/EIN"
## [22] "//IdRelatedOrgTxblPartnershipGrp/RelatedOrganizationName"
## [23] "//IdRelatedOrgTxblPartnershipGrp/PrimaryActivitiesTxt"
## [24] "//IdRelatedOrgTxblPartnershipGrp/LegalDomicileForeignCountryCd"
## [25] "//IdRelatedOrgTxblPartnershipGrp/LegalDomicileStateCd"
## [26] "//IdRelatedOrgTxblPartnershipGrp/DirectControllingNACd"
## [27] "//IdRelatedOrgTxblPartnershipGrp/DirectControllingEntityName"
## [28] "//IdRelatedOrgTxblPartnershipGrp/PredominantIncomeTypeTxt"
## [29] "//IdRelatedOrgTxblPartnershipGrp/ShareOfTotalIncomeAmt"
## [30] "//IdRelatedOrgTxblPartnershipGrp/ShareOfEOYAssetsAmt"
## [31] "//IdRelatedOrgTxblPartnershipGrp/DisproportionateAllocationsInd"
## [32] "//IdRelatedOrgTxblPartnershipGrp/UBICodeVAmt"
## [33] "//IdRelatedOrgTxblPartnershipGrp/GeneralOrManagingPartnerInd"
## [34] "//IdRelatedOrgTxblPartnershipGrp/OwnershipPct"# manual
if( ! is.null(table.headers) )
{table.headers}
# create new folder
dir.create( folder.name )
setwd( folder.name )
# create sample
set.seed( 1234 )
sample.index <- sample_n( index, sample.size )
write.csv( index, "sample-index.csv", row.names=F )
sample.urls <- sample.index$URL
# erase existing log files
file.create("XPATH-LOG.txt")
file.create("FAIL-LOG.txt")
start.build.time <- Sys.time() # --------------------
results.list <- list()
xpath.list <- list()
for( i in 1:length( sample.urls ) )
{
url <- sample.urls[i]
results.list[[i]] <-
build_rdb_table_v2( url,
table.name,
table.headers=table.headers )
xpath.list[[i]] <-
get_table_xpaths( url,
table.name,
table.headers=table.headers )
if( i %% 100 == 0 ){ print(i) }
}
end.build.time <- Sys.time() # --------------------
df <- dplyr::bind_rows( results.list )
fail.log <- readLines( "FAIL-LOG.txt" )
all.xpaths <- xpath.list %>% unlist() %>% unique() %>% sort()
cat( all.xpaths, sep="\n", file="XPATH-LOG.txt" )TABLE: SR-P03-T01-ID-RLTD-ORGS-TAXABLE-PARTNERSHIP
## [1] "NUMBER OF RETURNS PROCESSED: 1000"
## [1] "TOTAL NUMBER OF TABLE ROWS: 2822"TOTAL RUN TIME:
## Time difference of 43.54131 mins
## Time difference of -0.7256886 hourswrite.csv( df, paste0( folder.name, "/", table.name, ".csv" ), row.names=F )
head(df) Are all names standardized?
names(df)## [1] "OBJECT_ID" "EIN"
## [3] "NAME" "TAXYR"
## [5] "FORMTYPE" "URL"
## [7] "SR_03_RLTD_ORG_PTR_EIN" "SR_03_RLTD_ORG_PTR_PRIMARY_ACT"
## [9] "SR_03_RLTD_ORG_PTR_DMCL_STATE" "SR_03_RLTD_ORG_PTR_CTRL_NA"
## [11] "SR_03_RLTD_ORG_PTR_PREDMNTINCOME" "SR_03_RLTD_ORG_PTR_ALLOC_X"
## [13] "SR_03_RLTD_ORG_PTR_GENERAL_X" "SR_03_RLTD_ORG_PTR_NAME_L1"
## [15] "SR_03_RLTD_ORG_PTR_ADDR_L1" "SR_03_RLTD_ORG_PTR_ADDR_CITY"
## [17] "SR_03_RLTD_ORG_PTR_ADDR_STATE" "SR_03_RLTD_ORG_PTR_ADDR_ZIP"
## [19] "SR_03_RLTD_ORG_PTR_TOT_INCOME" "SR_03_RLTD_ORG_PTR_EOY_ASSETS"
## [21] "SR_03_RLTD_ORG_PTR_PCT_OWN" "SR_03_RLTD_ORG_PTR_NAME_L1.1"
## [23] "SR_03_RLTD_ORG_PTR_ADDR_L2" "SR_03_RLTD_ORG_PTR_NAME_L2"
## [25] "SR_03_RLTD_ORG_PTR_CODE_V_AMT" "SR_03_RLTD_ORG_PTR_DMCL_CNTR"
## [27] "SR_03_RLTD_ORG_PTR_ADDR_CNTR"
Check for xpaths that were used in the tables but do not exist in the Concordance file.
c.table <- dplyr::filter( concordance, rdb_table == table.name )
xpath.concordance <- c.table$xpath
setdiff( all.xpaths, xpath.concordance )## [1] ""
## [2] "/Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/DirectControllingEntityName"
## [3] "/Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress"
## [4] "/Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/RelatedOrganizationName"
## [5] "/Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress"Capture to update the Concordance file.
these <- setdiff( all.xpaths, xpath.concordance )
cat( these, sep="\n", file=paste0( folder.name, "/MISSING-XPATHS.txt" ) )Check the status of failed URLs.
if( length(fail.log) > 0 )
{
sapply( fail.log, get_message ) %>%
table() %>%
pander()
} else {
print("ALL URLs ARE VALID")
}## [1] "ALL URLs ARE VALID"
The create_edgelist() functions v1 and v2 are in R/utils.R.
V1 retains the root nodes Return and ReturnData.
V2 drops the root nodes and uses form versions “990” and “990EZ” as the root to visualize the sets of tables separately.
# KEEP ROOT NOTES Return & ReturnData
el <- create_edgelist_v1( all.xpaths )Edgelist format:
| V1 | V2 |
|---|---|
| Return | ReturnData |
| ReturnData | IRS990 |
| IRS990 | Form990PartVIISectionAGrp |
| Form990PartVIISectionAGrp | PersonNm |
| Form990PartVIISectionAGrp | TitleTxt |
| Form990PartVIISectionAGrp | AverageHoursPerWeekRt |
| Form990PartVIISectionAGrp | IndividualTrusteeOrDirectorInd |
| Form990PartVIISectionAGrp | OfficerInd |
| Form990PartVIISectionAGrp | ReportableCompFromOrgAmt |
| Form990PartVIISectionAGrp | ReportableCompFromRltdOrgAmt |
# library( data.tree )
nd <- FromDataFrameNetwork( network=el )
print( nd )## levelName
## 1 Return
## 2 °--ReturnData
## 3 °--IRS990ScheduleR
## 4 °--IdRelatedOrgTxblPartnershipGrp
## 5 ¦--DirectControllingEntityName
## 6 ¦ ¦--BusinessNameLine1Txt
## 7 ¦ °--BusinessNameLine2Txt
## 8 ¦--DirectControllingNACd
## 9 ¦--DisproportionateAllocationsInd
## 10 ¦--EIN
## 11 ¦--ForeignAddress
## 12 ¦ ¦--AddressLine1Txt
## 13 ¦ ¦--CityNm
## 14 ¦ ¦--CountryCd
## 15 ¦ ¦--ForeignPostalCd
## 16 ¦ °--ProvinceOrStateNm
## 17 ¦--GeneralOrManagingPartnerInd
## 18 ¦--LegalDomicileForeignCountryCd
## 19 ¦--LegalDomicileStateCd
## 20 ¦--OwnershipPct
## 21 ¦--PredominantIncomeTypeTxt
## 22 ¦--PrimaryActivitiesTxt
## 23 ¦--RelatedOrganizationName
## 24 ¦ °--BusinessNameLine1Txt
## 25 ¦--ShareOfEOYAssetsAmt
## 26 ¦--ShareOfTotalIncomeAmt
## 27 ¦--UBICodeVAmt
## 28 °--USAddress
## 29 ¦--AddressLine1Txt
## 30 ¦--AddressLine2Txt
## 31 ¦--CityNm
## 32 ¦--StateAbbreviationCd
## 33 °--ZIPCdSetGraphStyle( nd, rankdir = "LR")
SetEdgeStyle( nd, arrowhead = "vee", color = "grey20", penwidth = 2 )
SetNodeStyle( nd,
style = "filled,rounded",
shape = "box",
fillcolor = "LightBlue",
fontname = "helvetica",
fontcolor="black",
tooltip = GetDefaultTooltip )
# SetNodeStyle(acme$IT, fillcolor = "LightBlue", penwidth = "5px")
plot( nd )# library( networkD3 )
# net <- ToDataFrameNetwork( nd, "name" )
# simpleNetwork( net, fontSize = 12 )
simpleNetwork( el, fontSize = 12 )#plot with networkD3
radnet <- ToListExplicit( nd, unname = TRUE )
radialNetwork( radnet, fontSize = 12 )
t.xpaths <-
concordance %>%
filter( rdb_table == table.name ) %>%
select( xpath ) %>%
unique() %>%
arrange( xpath ) %>%
mutate( id=row_number() )
t.xpaths %>%
kable( align="l" )| xpath | id |
|---|---|
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressForeign/AddressLine1 | 1 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressForeign/AddressLine2 | 2 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressForeign/City | 3 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressForeign/Country | 4 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressForeign/PostalCode | 5 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressForeign/ProvinceOrState | 6 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressUS/AddressLine1 | 7 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressUS/AddressLine2 | 8 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressUS/City | 9 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressUS/State | 10 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/AddressUS/ZIPCode | 11 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/CodeVAmount | 12 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/DirectControllingEntity/NameBusiness/BusinessNameLine1 | 13 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/DirectControllingEntity/NameBusiness/BusinessNameLine2 | 14 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/DirectControllingEntity/NotApplicable | 15 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/DirectControllingEntityNA | 16 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/DirectControllingEntityName/BusinessNameLine1 | 17 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/DirectControllingEntityName/BusinessNameLine2 | 18 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/DisproportionateAllocations | 19 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/EIN | 20 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/GeneralOrManagingPartner | 21 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/LegalDomicile/ForeignCountry | 22 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/LegalDomicile/State | 23 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/LegalDomicileForeignCountry | 24 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/LegalDomicileState | 25 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/NameOfRelatedOrg/BusinessNameLine1 | 26 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/NameOfRelatedOrg/BusinessNameLine2 | 27 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/PercentageOwnership | 28 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/PredominantIncomeType | 29 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/PrimaryActivity | 30 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/ShareOfEOYAssets | 31 |
| /Return/ReturnData/IRS990ScheduleR/Form990ScheduleRPartIII/ShareOfTotalIncome | 32 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/DirectControllingEntityName/BusinessNameLine1 | 33 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/DirectControllingEntityName/BusinessNameLine1Txt | 34 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/DirectControllingEntityName/BusinessNameLine2 | 35 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/DirectControllingEntityName/BusinessNameLine2Txt | 36 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/DirectControllingNACd | 37 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/DisproportionateAllocationsInd | 38 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/EIN | 39 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/AddressLine1 | 40 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/AddressLine1Txt | 41 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/AddressLine2 | 42 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/AddressLine2Txt | 43 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/City | 44 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/CityNm | 45 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/Country | 46 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/CountryCd | 47 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/ForeignPostalCd | 48 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/PostalCode | 49 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/ProvinceOrState | 50 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ForeignAddress/ProvinceOrStateNm | 51 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/GeneralOrManagingPartnerInd | 52 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/LegalDomicileForeignCountryCd | 53 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/LegalDomicileStateCd | 54 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/OwnershipPct | 55 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/PredominantIncomeTypeTxt | 56 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/PrimaryActivitiesTxt | 57 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/RelatedOrganizationName/BusinessNameLine1 | 58 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/RelatedOrganizationName/BusinessNameLine1Txt | 59 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/RelatedOrganizationName/BusinessNameLine2 | 60 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/RelatedOrganizationName/BusinessNameLine2Txt | 61 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ShareOfEOYAssetsAmt | 62 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/ShareOfTotalIncomeAmt | 63 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/UBICodeVAmt | 64 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/AddressLine1 | 65 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/AddressLine1Txt | 66 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/AddressLine2 | 67 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/AddressLine2Txt | 68 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/City | 69 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/CityNm | 70 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/State | 71 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/StateAbbreviationCd | 72 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/ZIPCd | 73 |
| /Return/ReturnData/IRS990ScheduleR/IdRelatedOrgTxblPartnershipGrp/USAddress/ZIPCode | 74 |
Identify all unique groups used in xpaths.
if( is.null(table.headers) )
{ find_table_headers( table.name ) } else
table.headers## [1] "//Form990ScheduleRPartIII/AddressForeign"
## [2] "//Form990ScheduleRPartIII/AddressUS"
## [3] "//Form990ScheduleRPartIII/EIN"
## [4] "//Form990ScheduleRPartIII/NameOfRelatedOrg"
## [5] "//Form990ScheduleRPartIII/PrimaryActivity"
## [6] "//Form990ScheduleRPartIII/LegalDomicile"
## [7] "//Form990ScheduleRPartIII/LegalDomicileForeignCountry"
## [8] "//Form990ScheduleRPartIII/LegalDomicileState"
## [9] "//Form990ScheduleRPartIII/DirectControllingEntityNA"
## [10] "//Form990ScheduleRPartIII/DirectControllingEntity"
## [11] "//Form990ScheduleRPartIII/DirectControllingEntityName"
## [12] "//Form990ScheduleRPartIII/PredominantIncomeType"
## [13] "//Form990ScheduleRPartIII/ShareOfTotalIncome"
## [14] "//Form990ScheduleRPartIII/ShareOfEOYAssets"
## [15] "//Form990ScheduleRPartIII/DisproportionateAllocations"
## [16] "//Form990ScheduleRPartIII/CodeVAmount"
## [17] "//Form990ScheduleRPartIII/GeneralOrManagingPartner"
## [18] "//Form990ScheduleRPartIII/PercentageOwnership"
## [19] "//IdRelatedOrgTxblPartnershipGrp/ForeignAddress"
## [20] "//IdRelatedOrgTxblPartnershipGrp/USAddress"
## [21] "//IdRelatedOrgTxblPartnershipGrp/EIN"
## [22] "//IdRelatedOrgTxblPartnershipGrp/RelatedOrganizationName"
## [23] "//IdRelatedOrgTxblPartnershipGrp/PrimaryActivitiesTxt"
## [24] "//IdRelatedOrgTxblPartnershipGrp/LegalDomicileForeignCountryCd"
## [25] "//IdRelatedOrgTxblPartnershipGrp/LegalDomicileStateCd"
## [26] "//IdRelatedOrgTxblPartnershipGrp/DirectControllingNACd"
## [27] "//IdRelatedOrgTxblPartnershipGrp/DirectControllingEntityName"
## [28] "//IdRelatedOrgTxblPartnershipGrp/PredominantIncomeTypeTxt"
## [29] "//IdRelatedOrgTxblPartnershipGrp/ShareOfTotalIncomeAmt"
## [30] "//IdRelatedOrgTxblPartnershipGrp/ShareOfEOYAssetsAmt"
## [31] "//IdRelatedOrgTxblPartnershipGrp/DisproportionateAllocationsInd"
## [32] "//IdRelatedOrgTxblPartnershipGrp/UBICodeVAmt"
## [33] "//IdRelatedOrgTxblPartnershipGrp/GeneralOrManagingPartnerInd"
## [34] "//IdRelatedOrgTxblPartnershipGrp/OwnershipPct"Groups can then be used to extract the 1:M RDB tables:
if( is.null(table.headers) )
{ table.headers <- find_table_headers( table.name ) }
demo.url <- "https://nccs-efile.s3.us-east-1.amazonaws.com/xml/201913079349300206_public.xml"
doc <- xml2::read_xml( demo.url )
xml2::xml_ns_strip( doc )
df.demo <- get_table_v2( doc, table.name, table.headers )
df.demo ## NULLMapping of xml variable names to concordance variable names:
get_var_map( table.name=table.name )A table with proper names:
v.map <- get_var_map( table.name=table.name )
if( ! is.null(df.demo) )
{re_name( df.demo, v.map ) %>% as.data.frame()}
Demo of functions - demo data only.
[[1]]
<Form990PartVIISectionAGrp>
<PersonNm>
{text}
<TitleTxt>
{text}
<AverageHoursPerWeekRt>
{text}
<IndividualTrusteeOrDirectorInd>
{text}
<OfficerInd>
{text}
<ReportableCompFromOrgAmt>
{text}
<ReportableCompFromRltdOrgAmt>
{text}
<OtherCompensationAmt>
{text}node.list[[1]] %>% xml_parent() %>% xmltools::xml_view_trees() (1) ------------
+-- Form990PartVIISectionAGrp
+-- PersonNm
+-- TitleTxt
+-- AverageHoursPerWeekRt
+-- IndividualTrusteeOrDirectorInd
+-- OfficerInd
+-- ReportableCompFromOrgAmt
+-- ReportableCompFromRltdOrgAmt
++-- OtherCompensationAmt
(2) ------------
+-- Form990PartVIISectionAGrp
+-- PersonNm
+-- TitleTxt
+-- AverageHoursPerWeekRt
+-- IndividualTrusteeOrDirectorInd
+-- OfficerInd
+-- ReportableCompFromOrgAmt
+-- ReportableCompFromRltdOrgAmt
++-- OtherCompensationAmt