275 lines
9.3 KiB
Plaintext
275 lines
9.3 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Understanding the Kiva Dataset\n",
|
|
"\n",
|
|
"Before we actually get into the work of predicting anything based on the data Kiva makes public, we first want to get a better picture of what the dataset actually looks like.\n",
|
|
"\n",
|
|
"Our first step: What is the schema of the data? Spark SQL will make it easy to query data in the future, but we need to know first what is available."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"sparkSql = (SparkSession.builder\n",
|
|
" .master(\"local\")\n",
|
|
" .appName(\"Kiva Exploration\")\n",
|
|
" .getOrCreate())\n",
|
|
"\n",
|
|
"loans = sparkSql.read.format('json').load('kiva-data/loans.json')\n",
|
|
"lenders = sparkSql.read.format('json').load('kiva-data/lenders.json')\n",
|
|
"loans_lenders = sparkSql.read.format('json').load('kiva-data/loans_lenders.json')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"root\n",
|
|
" |-- activity: string (nullable = true)\n",
|
|
" |-- basket_amount: long (nullable = true)\n",
|
|
" |-- bonus_credit_eligibility: boolean (nullable = true)\n",
|
|
" |-- borrowers: array (nullable = true)\n",
|
|
" | |-- element: struct (containsNull = true)\n",
|
|
" | | |-- first_name: string (nullable = true)\n",
|
|
" | | |-- gender: string (nullable = true)\n",
|
|
" | | |-- last_name: string (nullable = true)\n",
|
|
" | | |-- pictured: boolean (nullable = true)\n",
|
|
" |-- currency_exchange_loss_amount: double (nullable = true)\n",
|
|
" |-- delinquent: boolean (nullable = true)\n",
|
|
" |-- description: struct (nullable = true)\n",
|
|
" | |-- languages: array (nullable = true)\n",
|
|
" | | |-- element: string (containsNull = true)\n",
|
|
" | |-- texts: struct (nullable = true)\n",
|
|
" | | |-- ar: string (nullable = true)\n",
|
|
" | | |-- en: string (nullable = true)\n",
|
|
" | | |-- es: string (nullable = true)\n",
|
|
" | | |-- fr: string (nullable = true)\n",
|
|
" | | |-- id: string (nullable = true)\n",
|
|
" | | |-- mn: string (nullable = true)\n",
|
|
" | | |-- pt: string (nullable = true)\n",
|
|
" | | |-- ru: string (nullable = true)\n",
|
|
" | | |-- vi: string (nullable = true)\n",
|
|
" |-- funded_amount: long (nullable = true)\n",
|
|
" |-- funded_date: string (nullable = true)\n",
|
|
" |-- id: long (nullable = true)\n",
|
|
" |-- image: struct (nullable = true)\n",
|
|
" | |-- id: long (nullable = true)\n",
|
|
" | |-- template_id: long (nullable = true)\n",
|
|
" |-- journal_totals: struct (nullable = true)\n",
|
|
" | |-- bulkEntries: long (nullable = true)\n",
|
|
" | |-- entries: long (nullable = true)\n",
|
|
" |-- lender_count: long (nullable = true)\n",
|
|
" |-- loan_amount: long (nullable = true)\n",
|
|
" |-- location: struct (nullable = true)\n",
|
|
" | |-- country: string (nullable = true)\n",
|
|
" | |-- country_code: string (nullable = true)\n",
|
|
" | |-- geo: struct (nullable = true)\n",
|
|
" | | |-- level: string (nullable = true)\n",
|
|
" | | |-- pairs: string (nullable = true)\n",
|
|
" | | |-- type: string (nullable = true)\n",
|
|
" | |-- town: string (nullable = true)\n",
|
|
" |-- name: string (nullable = true)\n",
|
|
" |-- paid_amount: double (nullable = true)\n",
|
|
" |-- paid_date: string (nullable = true)\n",
|
|
" |-- partner_id: long (nullable = true)\n",
|
|
" |-- payments: array (nullable = true)\n",
|
|
" | |-- element: struct (containsNull = true)\n",
|
|
" | | |-- amount: double (nullable = true)\n",
|
|
" | | |-- currency_exchange_loss_amount: double (nullable = true)\n",
|
|
" | | |-- local_amount: double (nullable = true)\n",
|
|
" | | |-- payment_id: long (nullable = true)\n",
|
|
" | | |-- processed_date: string (nullable = true)\n",
|
|
" | | |-- rounded_local_amount: double (nullable = true)\n",
|
|
" | | |-- settlement_date: string (nullable = true)\n",
|
|
" |-- planned_expiration_date: string (nullable = true)\n",
|
|
" |-- posted_date: string (nullable = true)\n",
|
|
" |-- sector: string (nullable = true)\n",
|
|
" |-- status: string (nullable = true)\n",
|
|
" |-- tags: array (nullable = true)\n",
|
|
" | |-- element: struct (containsNull = true)\n",
|
|
" | | |-- name: string (nullable = true)\n",
|
|
" |-- terms: struct (nullable = true)\n",
|
|
" | |-- disbursal_amount: double (nullable = true)\n",
|
|
" | |-- disbursal_currency: string (nullable = true)\n",
|
|
" | |-- disbursal_date: string (nullable = true)\n",
|
|
" | |-- loan_amount: long (nullable = true)\n",
|
|
" | |-- local_payments: array (nullable = true)\n",
|
|
" | | |-- element: struct (containsNull = true)\n",
|
|
" | | | |-- amount: double (nullable = true)\n",
|
|
" | | | |-- due_date: string (nullable = true)\n",
|
|
" | |-- loss_liability: struct (nullable = true)\n",
|
|
" | | |-- currency_exchange: string (nullable = true)\n",
|
|
" | | |-- currency_exchange_coverage_rate: double (nullable = true)\n",
|
|
" | | |-- nonpayment: string (nullable = true)\n",
|
|
" | |-- repayment_interval: string (nullable = true)\n",
|
|
" | |-- repayment_term: long (nullable = true)\n",
|
|
" | |-- scheduled_payments: array (nullable = true)\n",
|
|
" | | |-- element: struct (containsNull = true)\n",
|
|
" | | | |-- amount: double (nullable = true)\n",
|
|
" | | | |-- due_date: string (nullable = true)\n",
|
|
" |-- themes: array (nullable = true)\n",
|
|
" | |-- element: string (containsNull = true)\n",
|
|
" |-- translator: struct (nullable = true)\n",
|
|
" | |-- byline: string (nullable = true)\n",
|
|
" | |-- image: long (nullable = true)\n",
|
|
" |-- use: string (nullable = true)\n",
|
|
" |-- video: struct (nullable = true)\n",
|
|
" | |-- id: long (nullable = true)\n",
|
|
" | |-- thumbnailImageId: long (nullable = true)\n",
|
|
" | |-- title: string (nullable = true)\n",
|
|
" | |-- youtubeId: string (nullable = true)\n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"loans.printSchema()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"[Row(status=u'refunded', count=5504),\n",
|
|
" Row(status=u'defaulted', count=21776),\n",
|
|
" Row(status=u'in_repayment', count=155749),\n",
|
|
" Row(status=u'reviewed', count=3),\n",
|
|
" Row(status=u'deleted', count=2721),\n",
|
|
" Row(status=u'paid', count=775330),\n",
|
|
" Row(status=u'issue', count=199),\n",
|
|
" Row(status=u'inactive_expired', count=12421),\n",
|
|
" Row(status=u'fundraising', count=3986),\n",
|
|
" Row(status=u'expired', count=33773),\n",
|
|
" Row(status=u'inactive', count=2493),\n",
|
|
" Row(status=u'funded', count=173),\n",
|
|
" Row(status=u'', count=2)]"
|
|
]
|
|
},
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"loans.groupby(loans.status).count().collect()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"[Row(delinquent=None, count=970465), Row(delinquent=True, count=43665)]"
|
|
]
|
|
},
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"loans.groupby(loans.delinquent).count().collect()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"[Row(status=u'refunded', count=156),\n",
|
|
" Row(status=u'defaulted', count=20116),\n",
|
|
" Row(status=u'in_repayment', count=23393)]"
|
|
]
|
|
},
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"loans.where(loans.delinquent == True).groupby(loans.status).count().collect()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 19,
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"[Row(status=u'in_repayment')]"
|
|
]
|
|
},
|
|
"execution_count": 19,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"loans.registerTempTable('loans')\n",
|
|
"sparkSql.sql('''\n",
|
|
"SELECT loans.status\n",
|
|
"FROM loans\n",
|
|
"LIMIT 1\n",
|
|
"''').collect()"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 2",
|
|
"language": "python",
|
|
"name": "python2"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 2
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython2",
|
|
"version": "2.7.12"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 1
|
|
}
|