[Go] 부동산 크롤러

내가 내 집 매물 편하게 보려고 작성했다.
아직 추가해야할 기능도 많고 csv로 파싱해서 웹으로 띄우는게 목표이긴 하지만 나는 쓸만하게 쓰고있다

package main

import (
    "encoding/json"
    "fmt"
    "io/ioutil"
    "net/http"
)

type LandInfo struct {
    ArticleList []Articles `json:"articleList"`
}

type Articles struct {
    ArticleName        string   `json:"articleName"`
    FloorInfo          string   `json:"floorInfo"`
    DealOrWarrantPrc   string   `json:"dealOrWarrantPrc"`
    Direction          string   `json:"direction"`
    Area               int      `json:"area2"`
    ArticleFeatureDesc string   `json:"articleFeatureDesc"`
    TagList            []string `json:"tagList"`
    BuildingName       string   `json:"buildingName"`
    CpPcArticleURL     string   `json:"cpPcArticleUrl"`
}

func main() {

    url := "https://new.land.naver.com/api/articles/complex/2588?realEstateType=APT%3AABYG%3AJGC&tradeType=B1&tag=%3A%3A%3A%3A%3A%3A%3A%3A&rentPriceMin=0&rentPriceMax=900000000&priceMin=0&priceMax=33000&areaMin=0&areaMax=900000000&oldBuildYears&recentlyBuildYears&minHouseHoldCount&maxHouseHoldCount&showArticle=false&sameAddressGroup=true&minMaintenanceCost&maxMaintenanceCost&priceType=RETAIL&directions=&page=1&complexNo=2588&buildingNos=&areaNos=&type=list&order=rank"

    resp, err := http.Get(url)
    if err != nil {
        panic(err)
    }

    b, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        panic(err)
    }

    var data LandInfo
    json.Unmarshal(b, &data)
    fmt.Println("총 매물 수 : ", len(data.ArticleList))
    for i := 0; i < len(data.ArticleList); i++ {
        fmt.Println("=============================")
        fmt.Println("매물 : ", data.ArticleList[i].ArticleName, data.ArticleList[i].BuildingName)
        fmt.Println("층 : ", data.ArticleList[i].FloorInfo)
        fmt.Println("전세가 : ", data.ArticleList[i].DealOrWarrantPrc)
        fmt.Println("평수 : ", data.ArticleList[i].Area, "m²")
        fmt.Println("설명 : ", data.ArticleList[i].ArticleFeatureDesc)
        fmt.Println("링크 : ", data.ArticleList[i].CpPcArticleURL)
        fmt.Println("\nTags : ")
        for _, tag := range data.ArticleList[i].TagList {
            fmt.Println(" ", tag)
        }
        fmt.Println("=============================")
    }
}

결과

수정 버전

package main

import (
    "encoding/csv"
    "encoding/json"
    "fmt"
    "io/ioutil"
    "net/http"
    "os"
    "strings"
)

// 부동산 정보
type LandInfo struct {
    ArticleList []Articles `json:"articleList"`
}

// 부동산 정보 로우데이터
type Articles struct {
    ArticleName        string   `json:"articleName"`
    FloorInfo          string   `json:"floorInfo"`
    DealOrWarrantPrc   string   `json:"dealOrWarrantPrc"`
    Direction          string   `json:"direction"`
    Area               int      `json:"area2"`
    ArticleFeatureDesc string   `json:"articleFeatureDesc"`
    TagList            []string `json:"tagList"`
    BuildingName       string   `json:"buildingName"`
    CpPcArticleURL     string   `json:"cpPcArticleUrl"`
    ArticleConfirmYmd  string   `json:"articleConfirmYmd"`
    TradeTypeName      string   `json:"tradeTypeName"`
}

//getFromUrl 함수는 url을 받아서 LandInfo 형태의 struct로 반환한다.
func getFromUrl(url string) LandInfo {
    resp, err := http.Get(url)
    if err != nil {
        panic(err)
    }

    b, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        panic(err)
    }

    var data LandInfo
    json.Unmarshal(b, &data)
    return data
}

func printInfo(data LandInfo) {
    fmt.Println("총 매물 수 : ", len(data.ArticleList))
    for i := 0; i < len(data.ArticleList); i++ {
        fmt.Println("=============================")
        fmt.Println("매물 : ", data.ArticleList[i].ArticleName, data.ArticleList[i].BuildingName)
        fmt.Println("층 : ", data.ArticleList[i].FloorInfo)
        fmt.Println("전세가 : ", data.ArticleList[i].DealOrWarrantPrc)
        fmt.Println("평수 : ", data.ArticleList[i].Area, "m²")
        fmt.Println("설명 : ", data.ArticleList[i].ArticleFeatureDesc)
        fmt.Println("매물 확인 일자 : ", data.ArticleList[i].ArticleConfirmYmd)
        fmt.Println("링크 : ", data.ArticleList[i].CpPcArticleURL)
        fmt.Println("\nTags : ")
        for _, tag := range data.ArticleList[i].TagList {
            fmt.Println(" ", tag)
        }

    }
}

func convertToCSV(data LandInfo, wr *csv.Writer) {
    // Data 가공
    for _, d := range data.ArticleList {
        var row []string
        row = append(row, d.ArticleName)           //매물
        row = append(row, d.BuildingName)          //단지
        row = append(row, d.FloorInfo)             //층
        row = append(row, d.TradeTypeName)         //매물 형태
        row = append(row, d.DealOrWarrantPrc)      //금액
        row = append(row, fmt.Sprint(d.Area)+"m²") //평수
        row = append(row, d.ArticleFeatureDesc)    //설명

        //확인 일자 가공
        landDate := d.ArticleConfirmYmd[:4] + "-" + d.ArticleConfirmYmd[4:6] + "-" + d.ArticleConfirmYmd[6:]
        row = append(row, landDate)         //매물 확인 일자
        row = append(row, d.CpPcArticleURL) //링크

        // Tag 가공
        tags := strings.Join(d.TagList, ",") //비고
        row = append(row, tags)              //비고
        wr.Write(row)
    }

    wr.Flush()
}

func main() {
    file, _ := os.Create("2020-12-11.csv")
    wr := csv.NewWriter(file)

    // csv파일 헤더 입력
    header := []string{"매물", "단지", "층", "매물 형태", "금액", "평수", "설명", "매물 확인 일자", "링크", "비고"}
    wr.Write(header)

    // Data 가공할 URL 수집
    urls := []string{
        // 신나무실
        "https://new.land.naver.com/api/articles/complex/2588?realEstateType=APT%3AABYG%3AJGC&tradeType=B1&tag=%3A%3A%3A%3A%3A%3A%3A%3A&rentPriceMin=0&rentPriceMax=900000000&priceMin=0&priceMax=33000&areaMin=0&areaMax=900000000&oldBuildYears&recentlyBuildYears&minHouseHoldCount&maxHouseHoldCount&showArticle=false&sameAddressGroup=true&minMaintenanceCost&maxMaintenanceCost&priceType=RETAIL&directions=&page=1&complexNo=2588&buildingNos=&areaNos=&type=list&order=rank",
        // 벽적골 8단지 주공
        "https://new.land.naver.com/api/articles/complex/3988?realEstateType=APT%3AABYG%3AJGC&tradeType=B1&tag=%3A%3A%3A%3A%3A%3A%3A%3A&rentPriceMin=0&rentPriceMax=900000000&priceMin=0&priceMax=33000&areaMin=0&areaMax=900000000&oldBuildYears&recentlyBuildYears&minHouseHoldCount&maxHouseHoldCount&showArticle=false&sameAddressGroup=true&minMaintenanceCost&maxMaintenanceCost&priceType=RETAIL&directions=&page=1&complexNo=3988&buildingNos=&areaNos=1%3A3%3A2&type=list&order=rank",
        // 벽적골 롯데
        "https://new.land.naver.com/api/articles/complex/3638?realEstateType=APT%3AABYG%3AJGC&tradeType=B1&tag=%3A%3A%3A%3A%3A%3A%3A%3A&rentPriceMin=0&rentPriceMax=900000000&priceMin=0&priceMax=33000&areaMin=0&areaMax=900000000&oldBuildYears&recentlyBuildYears&minHouseHoldCount&maxHouseHoldCount&showArticle=false&sameAddressGroup=true&minMaintenanceCost&maxMaintenanceCost&priceType=RETAIL&directions=&page=1&complexNo=3638&buildingNos=&areaNos=3&type=list&order=rank",
        // 벽적골 우성
        "https://new.land.naver.com/api/articles/complex/12045?realEstateType=APT%3AABYG%3AJGC&tradeType=B1&tag=%3A%3A%3A%3A%3A%3A%3A%3A&rentPriceMin=0&rentPriceMax=900000000&priceMin=0&priceMax=33000&areaMin=0&areaMax=900000000&oldBuildYears&recentlyBuildYears&minHouseHoldCount&maxHouseHoldCount&showArticle=false&sameAddressGroup=true&minMaintenanceCost&maxMaintenanceCost&priceType=RETAIL&directions=&page=1&complexNo=12045&buildingNos=&areaNos=2%3A1&type=list&order=rank",
    }

    var datas LandInfo

    count := 0
    for count < len(urls) {
        datas = getFromUrl(urls[count])
        convertToCSV(datas, wr)
        count++
    }
}

csv파일 결과