Golang 爬取学院的安全测试题库

原文地址

前言

学院的安全教育测试题目好难，为了通过测试，也为了检验一下最近的学习成果，我尝试用 golang 爬取了整个安全题库，并保存成csv文件方便使用(轻松100分)

遇到的问题

• 寻找关键的 form-data

• json 的反序列化

实现

package main

import (
    "encoding/csv"
    "encoding/json"
    "fmt"
    "io/ioutil"
    "net/http"
    "net/url"
    "os"
    "strconv"
)

type ContentSlice struct {
    Content string `json:"content"`
    RightAnswer string `json:"rightAnswer"`
}

type Content struct{
    Content2 []ContentSlice `json:"content"`
}

type Data struct {
    Success bool `json:"success"`
    Data Content `json:"data"`
}

type Work struct {
    title string
    answer string

}

var (
    SeminarId = 137
    PageNum = 1
    sum=0
)

func main() {
    works := make([]Work, 0, 1000)
    for{
        var msg Data
        resp, err := http.PostForm("http://47.97.116.171:8085/web/getPractise",
            url.Values{"seminarId": {strconv.Itoa(SeminarId)}, "pageNum": {strconv.Itoa(PageNum)}})

        if err != nil {
            fmt.Println(err)
            break
        }

        body, err := ioutil.ReadAll(resp.Body)
        defer resp.Body.Close()

        json.Unmarshal([]byte(body), &msg)
        if err != nil {
            fmt.Println(err)
            break
        }
        if msg.Success{
            PageNum++
            sum++
            fmt.Println("select :",sum)
            works = append(works,Work{
                title:  msg.Data.Content2[0].Content,
                answer: msg.Data.Content2[0].RightAnswer,
            })

        }else{
            PageNum=1
            SeminarId++
            if SeminarId==150{
                fmt.Println("收集完成!!!")
                break
            }
        }

    }
    fmt.Println("total is ：",sum)
    f, _ := os.Create("Answers.csv")
    defer f.Close()

    f.WriteString("\xEF\xBB\xBF")

    writer := csv.NewWriter(f)
    defer writer.Flush()

    writer.Write([]string{"title", "answer"})
    for _, v := range works {
        writer.Write([]string{v.title, v.answer})
    }
}