最新消息:20210816 当前crifan.com域名已被污染,为防止失联,请关注(页面右下角的)公众号

【记录】go语言中实现http的POST且传递对应的post data

GO crifan 8748浏览 0评论

【背景】

折腾:

【记录】用go语言实现模拟登陆百度

期间,需要去发送POST操作,去模拟登陆百度,且post时要传递对应的post数据。

【折腾过程】

1.自己参考官网:

http://golang.org/pkg/net/http/

去看看,POST的参数和NewRequest的参数。

2.期间想要实现go中的函数的可选参数,默认参数,结果不支持:

【无法解决】go语言中实现函数的默认参数和可选参数

3.后来看了半天,很多帖子,貌似都是那个Form或PostForm的,但是感觉不太对。

4.参考:

Add custom HTTP header

看到是http.NewRequest设置为POST时,还传递了对应的postData了。

然后就去看看:

bytes.NewReader

的输入参数,是什么类型的。

http://golang.org/pkg/bytes/#NewReader

然后发现,就是普通的,byte[]

5.所以到目前为止,貌似明白了:

对于POST时所要传递的post data

是需要:

在http.NewRequest

http://golang.org/pkg/net/http/#NewRequest

时传入给:

body io.Reader

的,

但是:

http://golang.org/pkg/io/#Reader

不会用。

而别人传入的:

http://golang.org/pkg/bytes/#NewReader

还是基本能看懂的:

就是一个普通的reader,然后输入的是[]byte

而这个[]byte

是post data这string,被转换为对应的[]byte

而对应的post data的string,是key=value,中间通过&分隔开的。

需要自己,对于输入的postDict的键值对,自己组合出来。

或者是:

go中,或许也有对应的库函数去实现encode。

6.而之前看到:

http://golang.org/pkg/net/http/httputil/

看看有没有这样的工具。

不过貌似没看到。

但是看到其他的东西:

http://golang.org/pkg/net/http/httputil/#DumpRequest

可以帮你打印出来request,供调试用。

7.看到:

How to make an HTTP POST request in many languages

想到了:

go中,是不会用另外的url,去实现类似于

C#中的HttpUtility.UrlEncode?

所以去看看url:

http://golang.org/pkg/net/url/

然后终于找到所要的了:

http://golang.org/pkg/net/url/#Values.Encode

func (Values) Encode
func (v Values) Encode() string

Encode encodes the values into “URL encoded” form. e.g. "foo=bar&bar=baz"

8.然后就可以去尝试写POST相关代码了。

然后是用下面核心代码:

/*
 * [File]
 * EmulateLoginBaidu.go
 * 
 * [Function]
 * 【记录】用go语言实现模拟登陆百度
 * https://www.crifan.com/emulate_login_baidu_using_go_language/
 * 
 * [Version]
 * 2013-09-21
 *
 * [Contact]
 * https://www.crifan.com/about/me/
 */
package main

import (
    "fmt"
    "os"
    "runtime"
    "path"
    "strings"
    "time"
    //"io"
    "io/ioutil"
    "net/http"
    "net/http/cookiejar"
    "net/url"
    "bytes"
)


/***************************************************************************************************
    Global Variables
***************************************************************************************************/
var gCurCookies []*http.Cookie;
var gCurCookieJar *cookiejar.Jar;
var gLogger log4go.Logger;

/***************************************************************************************************
    Functions
***************************************************************************************************/
//do init before all others
func initAll(){
    gCurCookies = nil
    //var err error;
    gCurCookieJar,_ = cookiejar.New(nil)
    gLogger = nil
    
    //......
}

//get url response html
func getUrlRespHtml(strUrl string, postDict map[string]string) string{
    gLogger.Info("getUrlRespHtml, strUrl=%s", strUrl)
    gLogger.Info("postDict=%s", postDict)
    
    var respHtml string = "";
    
    httpClient := &http.Client{
        //Transport:nil,
        //CheckRedirect: nil,
        Jar:gCurCookieJar,
    }

    var httpReq *http.Request
    //var newReqErr error
    if nil == postDict {
        gLogger.Info("is GET")
        //httpReq, newReqErr = http.NewRequest("GET", strUrl, nil)
        httpReq, _ = http.NewRequest("GET", strUrl, nil)
        // ...
        //httpReq.Header.Add("If-None-Match", `W/"wyzzy"`)
    } else {
        gLogger.Info("is POST")
        postValues := url.Values{}
        for postKey, PostValue := range postDict{
            postValues.Set(postKey, PostValue)
        }
        gLogger.Info("postValues=%s", postValues)
        postDataStr := postValues.Encode()
        gLogger.Info("postDataStr=%s", postDataStr)
        postDataBytes := []byte(postDataStr)
        gLogger.Info("postDataBytes=%s", postDataBytes)
        postBytesReader := bytes.NewReader(postDataBytes)
        //httpReq, newReqErr = http.NewRequest("POST", strUrl, postBytesReader)
        httpReq, _ = http.NewRequest("POST", strUrl, postBytesReader)
        //httpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
        httpReq.Header.Add("Content-Type", "application/x-www-form-urlencoded")
    }
    
    httpResp, err := httpClient.Do(httpReq)
    // ...
    
    //httpResp, err := http.Get(strUrl)
    //gLogger.Info("http.Get done")
    if err != nil {
        gLogger.Warn("http get strUrl=%s response error=%s\n", strUrl, err.Error())
    }
    gLogger.Info("httpResp.Header=%s", httpResp.Header)
    gLogger.Debug("httpResp.Status=%s", httpResp.Status)

    defer httpResp.Body.Close()
    // gLogger.Info("defer httpResp.Body.Close done")
    
    body, errReadAll := ioutil.ReadAll(httpResp.Body)
    //gLogger.Info("ioutil.ReadAll done")
    if errReadAll != nil {
        gLogger.Warn("get response for strUrl=%s got error=%s\n", strUrl, errReadAll.Error())
    }
    //gLogger.Debug("body=%s\n", body)

    //gCurCookies = httpResp.Cookies()
    //gCurCookieJar = httpClient.Jar;
    gCurCookies = gCurCookieJar.Cookies(httpReq.URL);
    //gLogger.Info("httpResp.Cookies done")
    
    //respHtml = "just for test log ok or not"
    respHtml = string(body)
    //gLogger.Info("httpResp body []byte to string done")

    return respHtml
}

func dbgPrintCurCookies() {
    var cookieNum int = len(gCurCookies);
    gLogger.Info("cookieNum=%d", cookieNum)
    for i := 0; i < cookieNum; i++ {
        var curCk *http.Cookie = gCurCookies[i];
        //gLogger.Info("curCk.Raw=%s", curCk.Raw)
        gLogger.Info("------ Cookie [%d]------", i)
        gLogger.Info("Name\t\t=%s", curCk.Name)
        gLogger.Info("Value\t=%s", curCk.Value)
        gLogger.Info("Path\t\t=%s", curCk.Path)
        gLogger.Info("Domain\t=%s", curCk.Domain)
        gLogger.Info("Expires\t=%s", curCk.Expires)
        gLogger.Info("RawExpires\t=%s", curCk.RawExpires)
        gLogger.Info("MaxAge\t=%d", curCk.MaxAge)
        gLogger.Info("Secure\t=%t", curCk.Secure)
        gLogger.Info("HttpOnly\t=%t", curCk.HttpOnly)
        gLogger.Info("Raw\t\t=%s", curCk.Raw)
        gLogger.Info("Unparsed\t=%s", curCk.Unparsed)
    }
}

func main() {
    initAll()
    
    //......
    
    //step3: verify returned cookies
    if bGotCookieBaiduid && bExtractTokenValueOK {
        gLogger.Info("======步骤3:登陆百度并检验返回的Cookie ======");
        staticPageUrl := "http://www.baidu.com/cache/user/html/jump.html";
        
        postDict := map[string]string{}
        //postDict["ppui_logintime"] = ""
        postDict["charset"] = "utf-8"
        //postDict["codestring"] = ""
        postDict["token"] = strLoginToken
        postDict["isPhone"] = "false"
        postDict["index"] = "0"
        //postDict["u"] = ""
        //postDict["safeflg"] = "0"
        postDict["staticpage"] = staticPageUrl
        postDict["loginType"] = "1"
        postDict["tpl"] = "mn"
        postDict["callback"] = "parent.bdPass.api.login._postCallback"

        strBaiduUsername := ""
        strBaiduPassword := ""
        // stdinReader := bufio.NewReader(os.Stdin)
        // inputBytes, _ := stdinReader.ReadString('\n')
        // fmt.Printf("Input Char Is : %v", string([]byte(input)[0]))
        //_, err1 := fmt.Scanf("%s", &strBaiduUsername)
        //fmt.Println("Plese input:")
        //fmt.Println("Baidu Username:")
        gLogger.Info("Plese input:")
        gLogger.Info("Baidu Username:")
        _, err1 := fmt.Scanln(&strBaiduUsername)
        if nil == err1 {
            gLogger.Info("strBaiduUsername=%s", strBaiduUsername)
        }
        //fmt.Println("Baidu Password:")
        gLogger.Info("Baidu Password:")
        //_, err2 := fmt.Scanf("%s", &strBaiduPassword)
        _, err2 := fmt.Scanln(&strBaiduPassword)
        if nil == err2 {
            gLogger.Info("strBaiduPassword=%s", strBaiduPassword)
        }
        
        postDict["username"] = strBaiduUsername
        postDict["password"] = strBaiduPassword
        postDict["verifycode"] = ""
        postDict["mem_pass"] = "on"
        
        gLogger.Debug("postDict=%s", postDict)
        
        baiduMainLoginUrl := "https://passport.baidu.com/v2/api/?login";
        loginBaiduRespHtml := getUrlRespHtml(baiduMainLoginUrl, postDict);
        gLogger.Debug("loginBaiduRespHtml=%s", loginBaiduRespHtml)
    }

 }

实现了可以成功发送POST,传递进入post data:

[2013/09/21 18:26:42 ] [INFO] (main.main:294) ======步骤3:登陆百度并检验返回的Cookie ======
[2013/09/21 18:26:42 ] [INFO] (main.main:319) Plese input:
[2013/09/21 18:26:42 ] [INFO] (main.main:320) Baidu Username:
[2013/09/21 18:26:48 ] [INFO] (main.main:323) strBaiduUsername=xxxxxx
[2013/09/21 18:26:48 ] [INFO] (main.main:326) Baidu Password:
[2013/09/21 18:26:50 ] [INFO] (main.main:330) strBaiduPassword=yyyyyy
[2013/09/21 18:26:50 ] [DEBG] (main.main:338) postDict=map[charset:utf-8 isPhone:false index:0 tpl:mn username:xxxxxx verifycode: token:0933758100af2943e1948ea011386ac8 staticpage:http://www.baidu.com/cache/user/html/jump.html loginType:1 callback:parent.bdPass.api.login._postCallback password:yyyyyy mem_pass:on]
[2013/09/21 18:26:50 ] [INFO] (main.getUrlRespHtml:127) getUrlRespHtml, strUrl=https://passport.baidu.com/v2/api/?login
[2013/09/21 18:26:50 ] [INFO] (main.getUrlRespHtml:128) postDict=map[token:0933758100af2943e1948ea011386ac8 staticpage:http://www.baidu.com/cache/user/html/jump.html loginType:1 callback:parent.bdPass.api.login._postCallback password:yyyyyy mem_pass:on charset:utf-8 isPhone:false index:0 tpl:mn username:xxxxxx verifycode:]
[2013/09/21 18:26:50 ] [INFO] (main.getUrlRespHtml:147) is POST
[2013/09/21 18:26:50 ] [INFO] (main.getUrlRespHtml:152) postValues=map[token:[0933758100af2943e1948ea011386ac8] staticpage:[http://www.baidu.com/cache/user/html/jump.html] loginType:[1] callback:[parent.bdPass.api.login._postCallback] password:[yyyyyy] mem_pass:[on] charset:[utf-8] isPhone:[false] index:[0] tpl:[mn] username:[xxxxxx] verifycode:[]]
[2013/09/21 18:26:50 ] [INFO] (main.getUrlRespHtml:154) postDataStr=callback=parent.bdPass.api.login._postCallback&charset=utf-8&index=0&isPhone=false&loginType=1&mem_pass=on&password=yyyyyy&staticpage=http%3A%2F%2Fwww.baidu.com%2Fcache%2Fuser%2Fhtml%2Fjump.html&token=0933758100af2943e1948ea011386ac8&tpl=mn&username=xxxxxx&verifycode=
[2013/09/21 18:26:50 ] [INFO] (main.getUrlRespHtml:156) postDataBytes=callback=parent.bdPass.api.login._postCallback&charset=utf-8&index=0&isPhone=false&loginType=1&mem_pass=on&password=yyyyyy&staticpage=http%3A%2F%2Fwww.baidu.com%2Fcache%2Fuser%2Fhtml%2Fjump.html&token=0933758100af2943e1948ea011386ac8&tpl=mn&username=xxxxxx&verifycode=

然后模拟登陆百度,可以正常返回对应的各种cookie的:

[2013/09/21 18:26:50 ] [INFO] (main.getUrlRespHtml:172) httpResp.Header=map[Etag:[w/"Zv4yMMLJASqKOV17EWzXUpi3rNVEPvgZ:1379759203"] Set-Cookie:[BDUSS=G1LNG5uLTNYWkU2bzA2SGxCZHZ2Rm5ocnN-MEhFem5uQkZrdkJFVmplUmpBV1ZTQVFBQUFBJCQAAAAAAAAAAAEAAAB-OUgCYWdhaW5pbnB1dAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGN0PVJjdD1SM; expires=Wed, 08-Dec-2021 10:26:43 GMT; path=/; domain=baidu.com; httponly PTOKEN=deleted; expires=Fri, 21-Sep-2012 10:26:42 GMT; path=/; domain=baidu.com; httponly PTOKEN=0f1e0187b042630a47c4eea8e0e96a2f; expires=Wed, 08-Dec-2021 10:26:43 GMT; path=/; domain=passport.baidu.com; httponly STOKEN=8d6ce0cbc7f689a8cd647b8beb5872e3; expires=Wed, 08-Dec-2021 10:26:43 GMT; path=/; domain=passport.baidu.com; httponly SAVEUSERID=deleted; expires=Fri, 21-Sep-2012 10:26:42 GMT; path=/; domain=passport.baidu.com; httponly USERNAMETYPE=1; expires=Wed, 08-Dec-2021 10:26:43 GMT; path=/; domain=passport.baidu.com; httponly] Cache-Control:[public] Server:[] P3p:[CP=" OTI DSP COR IVA OUR IND COM "] Date:[Sat, 21 Sep 2013 10:26:43 GMT] Content-Type:[text/html] Connection:[keep-alive] Last-Modified:[Sat, 21 Sep 2013 10:26:43 10SepGMT] Pragma:[public] Expires:[0] Vary:[Accept-Encoding]]
[2013/09/21 18:26:50 ] [DEBG] (main.getUrlRespHtml:173) httpResp.Status=200 OK
[2013/09/21 18:26:50 ] [DEBG] (main.main:342) loginBaiduRespHtml=<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
</head>
<body>


<script type="text/javascript">

var url = encodeURI('http://www.baidu.com/cache/user/html/jump.html?hao123Param=RzFMTkc1dUxUTllXa1UyYnpBMlNHeENaSFoyUm01b2NuTi1NRWhGZW01dVFrWnJka0pGVm1wbFVtcEJWMVpUUVZGQlFVRkJKQ1FBQUFBQUFBQUFBQUVBQUFCLU9VZ0NZV2RoYVc1cGJuQjFkQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBR04wUFZKamREMVNN&callback=parent.bdPass.api.login._postCallback&index=0&codestring=&username=xxxxxx&phonenumber=&mail=&tpl=mn&u=https%3A%2F%2Fpassport.baidu.com%2F&needToModifyPassword=0&gotourl=&auth=&error=0');
//parent.callback(url)
window.location.replace(url);

</script>
</body>
</html>

如图:

can pass post data into http post resp cookie ok

至此,至少基本的POST,是搞定了。

 

【总结】

此处,对于http的POST,且要传递对应的post data的话,最最核心的代码是:

import (
    "net/http"
    "net/url"
    "bytes"
)

//get url response html
func getUrlRespHtml(strUrl string, postDict map[string]string) string{
    //......
    
    httpClient := &http.Client{
        //Transport:nil,
        //CheckRedirect: nil,
        Jar:gCurCookieJar,
    }

    var httpReq *http.Request
    //var newReqErr error
    if nil == postDict {
        gLogger.Info("is GET")
        //httpReq, newReqErr = http.NewRequest("GET", strUrl, nil)
        httpReq, _ = http.NewRequest("GET", strUrl, nil)
        // ...
        //httpReq.Header.Add("If-None-Match", `W/"wyzzy"`)
    } else {
        gLogger.Info("is POST")
        postValues := url.Values{}
        for postKey, PostValue := range postDict{
            postValues.Set(postKey, PostValue)
        }
        gLogger.Info("postValues=%s", postValues)
        postDataStr := postValues.Encode()
        gLogger.Info("postDataStr=%s", postDataStr)
        postDataBytes := []byte(postDataStr)
        gLogger.Info("postDataBytes=%s", postDataBytes)
        postBytesReader := bytes.NewReader(postDataBytes)
        //httpReq, newReqErr = http.NewRequest("POST", strUrl, postBytesReader)
        httpReq, _ = http.NewRequest("POST", strUrl, postBytesReader)
        //httpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
        httpReq.Header.Add("Content-Type", "application/x-www-form-urlencoded")
    }
    
    httpResp, err := httpClient.Do(httpReq)
    
    //......
}

func main() {
    //......
    
    postDict := map[string]string{}
    postDict["charset"] = "utf-8"
    postDict["token"] = strLoginToken
    postDict["isPhone"] = "false"
    postDict["index"] = "0"
    //......        
    
    baiduMainLoginUrl := "https://passport.baidu.com/v2/api/?login";
    loginBaiduRespHtml := getUrlRespHtml(baiduMainLoginUrl, postDict);
    gLogger.Debug("loginBaiduRespHtml=%s", loginBaiduRespHtml)
    
    //......

}

即可。

转载请注明:在路上 » 【记录】go语言中实现http的POST且传递对应的post data

发表我的评论
取消评论

表情

Hi,您需要填写昵称和邮箱!

  • 昵称 (必填)
  • 邮箱 (必填)
  • 网址
85 queries in 0.424 seconds, using 22.17MB memory