#
# Scrape data on house listings from realtor.com
#

# set working directory
setwd("~/notesofdabbler/Rspace/dayoh_housing/")

# load libraries
library(rvest)

## 
## Attaching package: 'rvest'
## 
## The following object is masked from 'package:utils':
## 
##     history

library(XML)


#
# Search URL with following filter applied
# 3+ bedrooms, 2+ baths, 1800+ sqft, 0-20 years old
#
srchurl="http://www.realtor.com/realestateandhomes-search/Centerville_OH/beds-3/baths-2/sqft-8/pfbm-10/show-hide-pending"

# using XML library
housedoc=htmlTreeParse(srchurl,useInternalNodes=TRUE)
ns_id=getNodeSet(housedoc,"//ul[@class='listing-summary']//li[@class='listing-location']//a[@href]") 
id=sapply(ns_id,function(x) xmlAttrs(x)["href"])
id

##                                                                                         href 
##                "/realestateandhomes-detail/3875-Mesquite-Dr_Beavercreek_OH_45440_1156581948" 
##                                                                                         href 
##                "/realestateandhomes-detail/5700-Troy-Pike_Huber-Heights_OH_45424_1154359505" 
##                                                                                         href 
##               "/realestateandhomes-detail/2158-Annandale-Pl_Beavercreek_OH_45385_1151654368" 
##                                                                                         href 
##              "/realestateandhomes-detail/4958-Wilmington-Pike_Kettering_OH_45440_1156044656" 
##                                                                                         href 
## "/realestateandhomes-detail/1404-Eden-Meadows-Way_Sugarcreek-Township_OH_45440_M47138-26737" 
##                                                                                         href 
##         "/realestateandhomes-detail/1719-Heritage-Lake-Dr_Centerville_OH_45458_M35086-04336" 
##                                                                                         href 
##              "/realestateandhomes-detail/3875-Mesquite-Dr_Beavercreek_OH_45440_M37209-50231" 
##                                                                                         href 
##                    "/realestateandhomes-detail/4322-Killian-Ct_Dayton_OH_45440_M47012-68114" 
##                                                                                         href 
##              "/realestateandhomes-detail/9620-Preserve-Pl_Centerville_OH_45458_M48704-43418" 
##                                                                                         href 
##  "/realestateandhomes-detail/1475-Country-Wood-Dr_Sugarcreek-Township_OH_45440_M47121-87036" 
##                                                                                         href 
##              "/realestateandhomes-detail/3848-Mesquite-Dr_Beavercreek_OH_45440_M37404-77585" 
##                                                                                         href 
##              "/realestateandhomes-detail/3766-Mesquite-Dr_Beavercreek_OH_45440_M44938-29951" 
##                                                                                         href 
## "/realestateandhomes-detail/1360-Eden-Meadows-Way_Sugarcreek-Township_OH_45440_M46918-39831" 
##                                                                                         href 
##           "/realestateandhomes-detail/1223-Ripplecreek-Ct_Centerville_OH_45458_M38951-86702" 
##                                                                                         href 
##   "/realestateandhomes-detail/986-Olde-Sterling-Way_Washington-Township_OH_45459_1152439949" 
##                                                                                         href 
##              "/realestateandhomes-detail/5873-Overbrooke-Rd_Centerville_OH_45440_1152054228" 
##                                                                                         href 
##                "/realestateandhomes-detail/5704-Oak-Valley-Rd_Kettering_OH_45440_1156042569" 
##                                                                                         href 
##            "/realestateandhomes-detail/512-Misty-Morning-Ct_Centerville_OH_45429_1155616027"

# using rvest library
housedoc = html(srchurl) 
houselist = housedoc %>% html_nodes(".listing-summary")
id =  houselist %>% html_node(".listing-location a") %>% html_attr("href")
id

##  [1] "/realestateandhomes-detail/226-Edgar-Ave_Dayton_OH_45410_1156432890"                       
##  [2] "/realestateandhomes-detail/1595-Little-Falls-Dr_Clearcreek-Township_OH_45458_1156544234"   
##  [3] "/realestateandhomes-detail/8811-Windbluff_Washington-Township_OH_45458_1156596059"         
##  [4] "/realestateandhomes-detail/3875-Mesquite-Dr_Beavercreek_OH_45440_1156581948"               
##  [5] "/realestateandhomes-detail/1404-Eden-Meadows-Way_Sugarcreek-Township_OH_45440_M47138-26737"
##  [6] "/realestateandhomes-detail/1719-Heritage-Lake-Dr_Centerville_OH_45458_M35086-04336"        
##  [7] "/realestateandhomes-detail/3875-Mesquite-Dr_Beavercreek_OH_45440_M37209-50231"             
##  [8] "/realestateandhomes-detail/4322-Killian-Ct_Dayton_OH_45440_M47012-68114"                   
##  [9] "/realestateandhomes-detail/9620-Preserve-Pl_Centerville_OH_45458_M48704-43418"             
## [10] "/realestateandhomes-detail/1475-Country-Wood-Dr_Sugarcreek-Township_OH_45440_M47121-87036" 
## [11] "/realestateandhomes-detail/3848-Mesquite-Dr_Beavercreek_OH_45440_M37404-77585"             
## [12] "/realestateandhomes-detail/3766-Mesquite-Dr_Beavercreek_OH_45440_M44938-29951"             
## [13] "/realestateandhomes-detail/1360-Eden-Meadows-Way_Sugarcreek-Township_OH_45440_M46918-39831"
## [14] "/realestateandhomes-detail/1223-Ripplecreek-Ct_Centerville_OH_45458_M38951-86702"          
## [15] "/realestateandhomes-detail/3875-Mesquite-Dr_Beavercreek_OH_45440_1156581554"               
## [16] "/realestateandhomes-detail/311-West-Plum-St_Tipp-City_OH_45371_1155924433"                 
## [17] "/realestateandhomes-detail/512-Misty-Morning-Ct_Centerville_OH_45429_1155616027"           
## [18] "/realestateandhomes-detail/986-Olde-Sterling-Way_Washington-Township_OH_45459_1152439949"