Setup

Load Packages

library( irs990efile )  # functions for parsing efile XMLs
library( dplyr )        # data wrangling
library( purrr )        # data wrangling 
library( pander )       # formatting
library( knitr )        # formatting
library( DT )           # table printing
library( httr )         # web requests 
library( xmltools )     # xml utilities
library( xml2 )         # xml utilities
library( XML )          # xml utilities 
library( data.tree )    # network visualization
library( networkD3 )    # network visualization
library( igraph )       # network visualization

source( "R/rdb-functions-v2.R")
source( "R/utils.R")

Data

head( index )

The Concordance File

The concordance is an RDB to XML Crosswalk that maps all xpaths onto a specified set of variables. It is located in the irs990efile package.


The full data dictionary is available on GitHub.

DATA DICTIONARY


concordance %>% 
  filter( rdb_table == table.name )  








xpath id
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/ApprovedByBoard 1
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/BalanceDue 2
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/Default 3
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/LoanFromOrganization 4
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/LoanToOrganization 5
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/NameBusiness/BusinessNameLine1 6
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/NameBusiness/BusinessNameLine2 7
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/NamePerson 8
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/OriginalPrincipalAmount 9
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/PurposeOfLoan 10
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/WrittenAgreement 11
/Return/ReturnData/IRS990ScheduleL/LoanTable/ApprovedByBoard 12
/Return/ReturnData/IRS990ScheduleL/LoanTable/BalanceDue 13
/Return/ReturnData/IRS990ScheduleL/LoanTable/Default 14
/Return/ReturnData/IRS990ScheduleL/LoanTable/LoanFromOrganization 15
/Return/ReturnData/IRS990ScheduleL/LoanTable/LoanToOrganization 16
/Return/ReturnData/IRS990ScheduleL/LoanTable/NameBusiness/BusinessNameLine1 17
/Return/ReturnData/IRS990ScheduleL/LoanTable/NameBusiness/BusinessNameLine2 18
/Return/ReturnData/IRS990ScheduleL/LoanTable/NamePerson 19
/Return/ReturnData/IRS990ScheduleL/LoanTable/OriginalPrincipalAmount 20
/Return/ReturnData/IRS990ScheduleL/LoanTable/PurposeOfLoan 21
/Return/ReturnData/IRS990ScheduleL/LoanTable/RelationshipWithOrganization 22
/Return/ReturnData/IRS990ScheduleL/LoanTable/WrittenAgreement 23
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/BalanceDueAmt 24
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/BoardOrCommitteeApprovalInd 25
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/BusinessName/BusinessNameLine1 26
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/BusinessName/BusinessNameLine1Txt 27
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/BusinessName/BusinessNameLine2 28
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/BusinessName/BusinessNameLine2Txt 29
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/DefaultInd 30
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/LoanFromOrganizationInd 31
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/LoanPurposeTxt 32
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/LoanToOrganizationInd 33
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/OriginalPrincipalAmt 34
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/PersonNm 35
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/RelationshipWithOrgTxt 36
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/WrittenAgreementInd 37
xpaths <- as.character( t.xpaths$xpath )
el <- create_edgelist_v1( xpaths )
nd <- FromDataFrameNetwork( network=el )
print( nd )
##                                           levelName
## 1  Return                                          
## 2   °--ReturnData                                  
## 3       °--IRS990ScheduleL                         
## 4           ¦--Form990ScheduleLPartII              
## 5           ¦   °--LoanTable                       
## 6           ¦       ¦--ApprovedByBoard             
## 7           ¦       ¦--BalanceDue                  
## 8           ¦       ¦--Default                     
## 9           ¦       ¦--LoanFromOrganization        
## 10          ¦       ¦--LoanToOrganization          
## 11          ¦       ¦--NameBusiness                
## 12          ¦       ¦   ¦--BusinessNameLine1       
## 13          ¦       ¦   °--BusinessNameLine2       
## 14          ¦       ¦--NamePerson                  
## 15          ¦       ¦--OriginalPrincipalAmount     
## 16          ¦       ¦--PurposeOfLoan               
## 17          ¦       ¦--WrittenAgreement            
## 18          ¦       °--RelationshipWithOrganization
## 19          ¦--LoanTable                           
## 20          ¦   ¦--ApprovedByBoard                 
## 21          ¦   ¦--BalanceDue                      
## 22          ¦   ¦--Default                         
## 23          ¦   ¦--LoanFromOrganization            
## 24          ¦   ¦--LoanToOrganization              
## 25          ¦   ¦--NameBusiness                    
## 26          ¦   ¦   ¦--BusinessNameLine1           
## 27          ¦   ¦   °--BusinessNameLine2           
## 28          ¦   ¦--NamePerson                      
## 29          ¦   ¦--OriginalPrincipalAmount         
## 30          ¦   ¦--PurposeOfLoan                   
## 31          ¦   ¦--WrittenAgreement                
## 32          ¦   °--RelationshipWithOrganization    
## 33          °--LoansBtwnOrgInterestedPrsnGrp       
## 34              ¦--BalanceDueAmt                   
## 35              ¦--BoardOrCommitteeApprovalInd     
## 36              ¦--BusinessName                    
## 37              ¦   ¦--BusinessNameLine1           
## 38              ¦   ¦--BusinessNameLine1Txt        
## 39              ¦   ¦--BusinessNameLine2           
## 40              ¦   °--BusinessNameLine2Txt        
## 41              ¦--DefaultInd                      
## 42              ¦--LoanFromOrganizationInd         
## 43              ¦--LoanPurposeTxt                  
## 44              ¦--LoanToOrganizationInd           
## 45              ¦--OriginalPrincipalAmt            
## 46              ¦--PersonNm                        
## 47              ¦--RelationshipWithOrgTxt          
## 48              °--WrittenAgreementInd
SetGraphStyle( nd, rankdir = "LR")
SetEdgeStyle( nd, arrowhead = "vee", color = "grey20", penwidth = 2 )
SetNodeStyle( nd, 
              style = "filled,rounded", 
              shape = "box", 
              fillcolor = "LightBlue", 
              fontname = "helvetica", 
              fontcolor="black",
              tooltip = GetDefaultTooltip )

# SetNodeStyle(acme$IT, fillcolor = "LightBlue", penwidth = "5px")
plot( nd )
# old version
find_group_names( table.name )
## [1] "//LoansBtwnOrgInterestedPrsnGrp" "//Form990ScheduleLPartII"       
## [3] "//LoanTable"
# new version
find_table_headers( table.name )
##  [1] "//LoansBtwnOrgInterestedPrsnGrp/BusinessName"               
##  [2] "//LoansBtwnOrgInterestedPrsnGrp/PersonNm"                   
##  [3] "//LoansBtwnOrgInterestedPrsnGrp/RelationshipWithOrgTxt"     
##  [4] "//LoansBtwnOrgInterestedPrsnGrp/LoanPurposeTxt"             
##  [5] "//LoansBtwnOrgInterestedPrsnGrp/LoanFromOrganizationInd"    
##  [6] "//LoansBtwnOrgInterestedPrsnGrp/LoanToOrganizationInd"      
##  [7] "//LoansBtwnOrgInterestedPrsnGrp/OriginalPrincipalAmt"       
##  [8] "//LoansBtwnOrgInterestedPrsnGrp/BalanceDueAmt"              
##  [9] "//LoansBtwnOrgInterestedPrsnGrp/DefaultInd"                 
## [10] "//LoansBtwnOrgInterestedPrsnGrp/BoardOrCommitteeApprovalInd"
## [11] "//LoansBtwnOrgInterestedPrsnGrp/WrittenAgreementInd"        
## [12] "//Form990ScheduleLPartII/LoanTable"                         
## [13] "//Form990ScheduleLPartII/TotalBalanceDue"                   
## [14] "//LoanTable/Form990ScheduleLPartII"                         
## [15] "//LoanTable/LoanTable"
# manual
if( ! is.null(table.headers) )
{table.headers}








Build the Table

# create new folder
dir.create( folder.name )
setwd( folder.name )

# create sample
set.seed( 1234 )
sample.index <- sample_n( index, sample.size )
write.csv( index, "sample-index.csv", row.names=F )
sample.urls <- sample.index$URL

# erase existing log files
file.create("XPATH-LOG.txt") 
file.create("FAIL-LOG.txt")


start.build.time <- Sys.time()    # --------------------

results.list <- list()
xpath.list <- list()


for( i in 1:length( sample.urls ) )
{
  
  url <- sample.urls[i]
  
  results.list[[i]] <- 
    build_rdb_table_v2( url, 
                        table.name, 
                        table.headers=table.headers )
  
  xpath.list[[i]]   <- 
    get_table_xpaths( url, 
                      table.name, 
                      table.headers=table.headers )
  
  if( i %% 100 == 0 ){ print(i) }
  
}


end.build.time <- Sys.time()      # --------------------


df <- dplyr::bind_rows( results.list )


fail.log <- readLines( "FAIL-LOG.txt" )
all.xpaths <- xpath.list %>% unlist() %>% unique() %>% sort()
cat( all.xpaths, sep="\n", file="XPATH-LOG.txt" )

TABLE: SL-P02-T01-LOANS-INTERESTED-PERS

## [1] "NUMBER OF RETURNS PROCESSED:   1000"
## [1] "TOTAL NUMBER OF TABLE ROWS:   798"

TOTAL RUN TIME:

## Time difference of 26.97826 mins
## Time difference of -0.4496377 hours

Write to File

write.csv( df, paste0( folder.name, "/", table.name, ".csv" ), row.names=F )
head(df) 

Are all names standardized?

names(df)
##  [1] "OBJECT_ID"                       "EIN"                            
##  [3] "NAME"                            "TAXYR"                          
##  [5] "FORMTYPE"                        "URL"                            
##  [7] "SL_02_LOAN_INT_NAME_PERS"        "SL_02_LOAN_RELATIONSHIP"        
##  [9] "SL_02_LOAN_PURPOSE"              "SL_02_LOAN_TO_ORG_X"            
## [11] "SL_02_LOAN_AMT_ORIGIN_PRINCIPAL" "SL_02_LOAN_BALANCE_DUE"         
## [13] "SL_02_LOAN_DEFAULT_X"            "SL_02_LOAN_BOARD_APPROV_X"      
## [15] "SL_02_LOAN_WRITTEN_AGREEMENT_X"  "SL_02_LOAN_FROM_ORG_X"          
## [17] "SL_02_LOAN_INT_NAME_ORG_L1"      "SL_02_LOAN_INT_NAME_ORG_L2"








Diagnostics

Missing or Extra XPATHS

Check for xpaths that were used in the tables but do not exist in the Concordance file.

c.table <- dplyr::filter( concordance, rdb_table == table.name ) 
xpath.concordance <- c.table$xpath
setdiff( all.xpaths, xpath.concordance )
## [1] ""                                                                             
## [2] "/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/BusinessName"

Capture to update the Concordance file.

these <- setdiff( all.xpaths, xpath.concordance )
cat( these, sep="\n", file=paste0( folder.name, "/MISSING-XPATHS.txt" ) )

Failes URLs

Check the status of failed URLs.

if( length(fail.log) > 0 )
{ 
  sapply( fail.log, get_message ) %>% 
  table() %>% 
  pander() 
} else {
  print("ALL URLs ARE VALID")
}
## [1] "ALL URLs ARE VALID"








XML Table Structure

Create Network Edgelists

The create_edgelist() functions v1 and v2 are in R/utils.R.

V1 retains the root nodes Return and ReturnData.

V2 drops the root nodes and uses form versions “990” and “990EZ” as the root to visualize the sets of tables separately.

# KEEP ROOT NOTES Return & ReturnData
el <- create_edgelist_v1( all.xpaths )



Edgelist format:

V1 V2
Return ReturnData
ReturnData IRS990
IRS990 Form990PartVIISectionAGrp
Form990PartVIISectionAGrp PersonNm
Form990PartVIISectionAGrp TitleTxt
Form990PartVIISectionAGrp AverageHoursPerWeekRt
Form990PartVIISectionAGrp IndividualTrusteeOrDirectorInd
Form990PartVIISectionAGrp OfficerInd
Form990PartVIISectionAGrp ReportableCompFromOrgAmt
Form990PartVIISectionAGrp ReportableCompFromRltdOrgAmt








Visualize Tree Structure

# library( data.tree )
nd <- FromDataFrameNetwork( network=el )
print( nd )
##                                      levelName
## 1  Return                                     
## 2   °--ReturnData                             
## 3       °--IRS990ScheduleL                    
## 4           °--LoansBtwnOrgInterestedPrsnGrp  
## 5               ¦--BalanceDueAmt              
## 6               ¦--BoardOrCommitteeApprovalInd
## 7               ¦--BusinessName               
## 8               ¦   ¦--BusinessNameLine1Txt   
## 9               ¦   °--BusinessNameLine2Txt   
## 10              ¦--DefaultInd                 
## 11              ¦--LoanFromOrganizationInd    
## 12              ¦--LoanPurposeTxt             
## 13              ¦--LoanToOrganizationInd      
## 14              ¦--OriginalPrincipalAmt       
## 15              ¦--PersonNm                   
## 16              ¦--RelationshipWithOrgTxt     
## 17              °--WrittenAgreementInd
SetGraphStyle( nd, rankdir = "LR")
SetEdgeStyle( nd, arrowhead = "vee", color = "grey20", penwidth = 2 )
SetNodeStyle( nd, 
              style = "filled,rounded", 
              shape = "box", 
              fillcolor = "LightBlue", 
              fontname = "helvetica", 
              fontcolor="black",
              tooltip = GetDefaultTooltip )

# SetNodeStyle(acme$IT, fillcolor = "LightBlue", penwidth = "5px")
plot( nd )
# library( networkD3 )
# net <- ToDataFrameNetwork( nd, "name" )
# simpleNetwork( net, fontSize = 12 )
simpleNetwork( el, fontSize = 12 )
#plot with networkD3
radnet <- ToListExplicit( nd, unname = TRUE )
radialNetwork( radnet, fontSize = 12 )








XPATH Summary

Unique Xpaths for this RDB Table

t.xpaths <- 
  concordance %>%
  filter( rdb_table == table.name ) %>% 
  select( xpath ) %>% 
  unique() %>% 
  arrange( xpath ) %>% 
  mutate( id=row_number() ) 

t.xpaths %>% 
  kable( align="l" )
xpath id
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/ApprovedByBoard 1
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/BalanceDue 2
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/Default 3
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/LoanFromOrganization 4
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/LoanToOrganization 5
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/NameBusiness/BusinessNameLine1 6
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/NameBusiness/BusinessNameLine2 7
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/NamePerson 8
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/OriginalPrincipalAmount 9
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/PurposeOfLoan 10
/Return/ReturnData/IRS990ScheduleL/Form990ScheduleLPartII/LoanTable/WrittenAgreement 11
/Return/ReturnData/IRS990ScheduleL/LoanTable/ApprovedByBoard 12
/Return/ReturnData/IRS990ScheduleL/LoanTable/BalanceDue 13
/Return/ReturnData/IRS990ScheduleL/LoanTable/Default 14
/Return/ReturnData/IRS990ScheduleL/LoanTable/LoanFromOrganization 15
/Return/ReturnData/IRS990ScheduleL/LoanTable/LoanToOrganization 16
/Return/ReturnData/IRS990ScheduleL/LoanTable/NameBusiness/BusinessNameLine1 17
/Return/ReturnData/IRS990ScheduleL/LoanTable/NameBusiness/BusinessNameLine2 18
/Return/ReturnData/IRS990ScheduleL/LoanTable/NamePerson 19
/Return/ReturnData/IRS990ScheduleL/LoanTable/OriginalPrincipalAmount 20
/Return/ReturnData/IRS990ScheduleL/LoanTable/PurposeOfLoan 21
/Return/ReturnData/IRS990ScheduleL/LoanTable/RelationshipWithOrganization 22
/Return/ReturnData/IRS990ScheduleL/LoanTable/WrittenAgreement 23
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/BalanceDueAmt 24
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/BoardOrCommitteeApprovalInd 25
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/BusinessName/BusinessNameLine1 26
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/BusinessName/BusinessNameLine1Txt 27
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/BusinessName/BusinessNameLine2 28
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/BusinessName/BusinessNameLine2Txt 29
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/DefaultInd 30
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/LoanFromOrganizationInd 31
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/LoanPurposeTxt 32
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/LoanToOrganizationInd 33
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/OriginalPrincipalAmt 34
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/PersonNm 35
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/RelationshipWithOrgTxt 36
/Return/ReturnData/IRS990ScheduleL/LoansBtwnOrgInterestedPrsnGrp/WrittenAgreementInd 37

Grouping Variables

Identify all unique groups used in xpaths.

if( is.null(table.headers) )
{ find_table_headers( table.name ) } else
table.headers
##  [1] "//LoansBtwnOrgInterestedPrsnGrp/BusinessName"               
##  [2] "//LoansBtwnOrgInterestedPrsnGrp/PersonNm"                   
##  [3] "//LoansBtwnOrgInterestedPrsnGrp/RelationshipWithOrgTxt"     
##  [4] "//LoansBtwnOrgInterestedPrsnGrp/LoanPurposeTxt"             
##  [5] "//LoansBtwnOrgInterestedPrsnGrp/LoanFromOrganizationInd"    
##  [6] "//LoansBtwnOrgInterestedPrsnGrp/LoanToOrganizationInd"      
##  [7] "//LoansBtwnOrgInterestedPrsnGrp/OriginalPrincipalAmt"       
##  [8] "//LoansBtwnOrgInterestedPrsnGrp/BalanceDueAmt"              
##  [9] "//LoansBtwnOrgInterestedPrsnGrp/DefaultInd"                 
## [10] "//LoansBtwnOrgInterestedPrsnGrp/BoardOrCommitteeApprovalInd"
## [11] "//LoansBtwnOrgInterestedPrsnGrp/WrittenAgreementInd"        
## [12] "//Form990ScheduleLPartII/LoanTable"                         
## [13] "//Form990ScheduleLPartII/TotalBalanceDue"                   
## [14] "//LoanTable/Form990ScheduleLPartII"                         
## [15] "//LoanTable/LoanTable"

Groups can then be used to extract the 1:M RDB tables:

if( is.null(table.headers) )
{ table.headers <- find_table_headers( table.name ) }
demo.url <- "https://nccs-efile.s3.us-east-1.amazonaws.com/xml/201913079349300206_public.xml"
doc <- xml2::read_xml( demo.url )
xml2::xml_ns_strip( doc )
df.demo <- get_table_v2( doc, table.name, table.headers )
df.demo 
## NULL

Standardize Variable Names

Mapping of xml variable names to concordance variable names:

get_var_map( table.name=table.name )

A table with proper names:

v.map <- get_var_map( table.name=table.name )

if( ! is.null(df.demo) )
{re_name( df.demo, v.map ) %>% as.data.frame()}