5. crifanLib.cs中的全局变量,初始化代码,私有函数

此处,顺便也把对应的,全局变量,初始化代码,私有函数等等,贴出来,供参考:


    public struct pairItem
    {
        public string key;
        public string value;
    };

    private Dictionary<string, DateTime> calcTimeList;

    const char replacedChar = '_';

    string[] cookieFieldArr = { "expires", "domain", "secure", "path", "httponly", "version" };

    //IE7
    const string constUserAgent_IE7_x64 = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E)";
    //IE8
    const string constUserAgent_IE8_x64 = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E";
    //IE9
    const string constUserAgent_IE9_x64 = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)"; // x64
    const string constUserAgent_IE9_x86 = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"; // x86
    //Chrome
    const string constUserAgent_Chrome = "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.99 Safari/533.4";
    //Mozilla Firefox
    const string constUserAgent_Firefox = "Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:1.9.2.6) Gecko/20100625 Firefox/3.6.6";
    private string gUserAgent;

    private WebProxy gProxy = null;

    //detault values:
    //getUrlResponse
    private const Dictionary<string, string> defHeaderDict = null;
    private const Dictionary<string, string> defPostDict = null;
    private const int defTimeout = 30 * 1000;
    private const string defPostDataStr = null;
    private const int defReadWriteTimeout = 30 * 1000;
    //getUrlRespHtml 
    private const string defCharset = null;
    //getUrlRespHtml_multiTry
    private const int defMaxTryNum = 5;
    private const int defRetryFailSleepTime = 100; //sleep time in ms when retry fail for getUrlRespHtml

    List<string> cookieFieldList = new List<string>();

    CookieCollection curCookies = null;

    //private long totalLength = 0;
    //private long currentLength = 0;
#if USE_GETURLRESPONSE_BW
    //indicate background worker complete or not
    bool bNotCompleted_resp = true;
    //store response of http request
    private HttpWebResponse gCurResp = null;
#endif

    private BackgroundWorker gBgwDownload;
    //indicate download complete or not
    bool bNotCompleted_download = true;
    //store current read out data len
    private int gRealReadoutLen = 0;
    Action<int> gFuncUpdateProgress = null;

    public crifanLib()
    {
        //!!! for load embedded dll: (1) register resovle handler
        AppDomain.CurrentDomain.AssemblyResolve += new ResolveEventHandler(CurrentDomain_AssemblyResolve);

        //http related
        gUserAgent = constUserAgent_IE8_x64;
        //set max enough to avoid http request is used out -> avoid dead while get response 
        System.Net.ServicePointManager.DefaultConnectionLimit = 200;

        curCookies = new CookieCollection();
        // init const cookie keys
        foreach (string key in cookieFieldArr)
        {
            cookieFieldList.Add(key);
        }

        //init for calc time
        calcTimeList = new Dictionary<string, DateTime>();
#if USE_GETURLRESPONSE_BW
        gBgwDownload = new BackgroundWorker();
#endif

        //debug
        //gProxy = new WebProxy("127.0.0.1", 8087);
    }

    /*------------------------Private Functions------------------------------*/

    //!!! for load embedded dll: (2) implement this handler
    System.Reflection.Assembly CurrentDomain_AssemblyResolve(object sender, ResolveEventArgs args)
    {
        string dllName = args.Name.Contains(",") ? args.Name.Substring(0, args.Name.IndexOf(',')) : args.Name.Replace(".dll", "");

        dllName = dllName.Replace(".", "_");

        if (dllName.EndsWith("_resources")) return null;

        System.Resources.ResourceManager rm = new System.Resources.ResourceManager(GetType().Namespace + ".Properties.Resources", System.Reflection.Assembly.GetExecutingAssembly());

        byte[] bytes = (byte[])rm.GetObject(dllName);

        return System.Reflection.Assembly.Load(bytes);
    }

    // replace the replacedChar back to original ','
    private string _recoverExpireField(Match foundPprocessedExpire)
    {
        string recovedStr = "";
        recovedStr = foundPprocessedExpire.Value.Replace(replacedChar, ',');
        return recovedStr;
    }

    //replace ',' with replacedChar
    private string _processExpireField(Match foundExpire)
    {
        string replacedComma = "";
        replacedComma = foundExpire.Value.ToString().Replace(',', replacedChar);
        return replacedComma;
    }

    //replace "0A" (in \x0A) into '\n'
    private string _replaceEscapeSequenceToChar(Match foundEscapeSequence)
    {
        char[] hexValues = new char[2];
        //string hexChars = foundEscapeSequence.Value.ToString();
        string matchedEscape = foundEscapeSequence.ToString();
        hexValues[0] = matchedEscape[2];
        hexValues[1] = matchedEscape[3];
        string hexValueString = new string(hexValues);
        int convertedInt = int.Parse(hexValueString, NumberStyles.HexNumber, NumberFormatInfo.InvariantInfo);
        char hexChar = Convert.ToChar(convertedInt);
        string hexStr = hexChar.ToString();
        return hexStr;
    }
    
    //check whether need add/retain this cookie
    // not add for:
    // ck is null or ck name is null
    // domain is null and curDomain is not set
    // expired and retainExpiredCookie==false
    private bool needAddThisCookie(Cookie ck, string curDomain)
    {
        bool needAdd = false;

        if ((ck == null) || (ck.Name == ""))
        {
            needAdd = false;
        }
        else
        {
            if (ck.Domain != "")
            {
                needAdd = true;
            }
            else// ck.Domain == ""
            {
                if (curDomain != "")
                {
                    ck.Domain = curDomain;
                    needAdd = true;
                }
                else // curDomain == ""
                {
                    // not set current domain, omit this
                    // should not add empty domain cookie, for this will lead execute CookieContainer.Add() fail !!!
                    needAdd = false;
                }
            }
        }

        return needAdd;
    }

    //quote the input dict values
    //note: the return result for first para no '&'
    private string _quoteParas(Dictionary<string, string> paras, bool spaceToPercent20 = true)
    {
        string quotedParas = "";
        bool isFirst = true;
        string val = "";
        foreach (string para in paras.Keys)
        {
            if (paras.TryGetValue(para, out val))
            {
                string encodedVal = "";
                if (spaceToPercent20)
                {
                    //encodedVal = HttpUtility.UrlPathEncode(val);
                    //encodedVal = Uri.EscapeDataString(val);
                    //encodedVal = Uri.EscapeUriString(val);
                    encodedVal = HttpUtility.UrlEncode(val).Replace("+", "%20");
                }
                else
                {
                    encodedVal = HttpUtility.UrlEncode(val); //space to +
                }

                if (isFirst)
                {
                    isFirst = false;
                    quotedParas += para + "=" + encodedVal;
                }
                else
                {
                    quotedParas += "&" + para + "=" + encodedVal;
                }
            }
            else
            {
                break;
            }
        }

        return quotedParas;
    }

    /* get url's response
     * */
    private HttpWebResponse _getUrlResponse(string url,
                                    Dictionary<string, string> headerDict = defHeaderDict,
                                    Dictionary<string, string> postDict = defPostDict,
                                    int timeout = defTimeout,
                                    string postDataStr = defPostDataStr,
                                    int readWriteTimeout = defReadWriteTimeout)
    {
        //CookieCollection parsedCookies;

        HttpWebResponse resp = null;

        HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);

        req.AllowAutoRedirect = true;
        req.Accept = "*/*";

        //req.ContentType = "text/plain";

        //const string gAcceptLanguage = "en-US"; // zh-CN/en-US
        //req.Headers["Accept-Language"] = gAcceptLanguage;

        req.KeepAlive = true;

        req.UserAgent = gUserAgent;

        req.Headers["Accept-Encoding"] = "gzip, deflate";
        //req.AutomaticDecompression = DecompressionMethods.GZip;
        req.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;

        req.Proxy = gProxy;

        if (timeout > 0)
        {
            req.Timeout = timeout;
        }

        if (readWriteTimeout > 0)
        {
            //default ReadWriteTimeout is 300000=300 seconds = 5 minutes !!!
            //too long, so here change to 300000 = 30 seconds
            //for support TimeOut for later StreamReader's ReadToEnd
            req.ReadWriteTimeout = readWriteTimeout;
        }

        if (curCookies != null)
        {
            req.CookieContainer = new CookieContainer();
            req.CookieContainer.PerDomainCapacity = 40; // following will exceed max default 20 cookie per domain
            req.CookieContainer.Add(curCookies);
        }

        if ((headerDict != null) && (headerDict.Count > 0))
        {
            foreach (string header in headerDict.Keys)
            {
                string headerValue = "";
                if (headerDict.TryGetValue(header, out headerValue))
                {
                    string lowecaseHeader = header.ToLower();
                    // following are allow the caller overwrite the default header setting
                    if (lowecaseHeader == "referer")
                    {
                        req.Referer = headerValue;
                    }
                    else if (
                            (lowecaseHeader == "allow-autoredirect") ||
                            (lowecaseHeader == "allowautoredirect") ||
                            (lowecaseHeader == "allow autoredirect")
                            )
                    {
                        bool isAllow = false;
                        if (bool.TryParse(headerValue, out isAllow))
                        {
                            req.AllowAutoRedirect = isAllow;
                        }
                    }
                    else if (lowecaseHeader == "accept")
                    {
                        req.Accept = headerValue;
                    }
                    else if (
                            (lowecaseHeader == "keep-alive") ||
                            (lowecaseHeader == "keepalive") ||
                            (lowecaseHeader == "keep alive")
                            )
                    {
                        bool isKeepAlive = false;
                        if (bool.TryParse(headerValue, out isKeepAlive))
                        {
                            req.KeepAlive = isKeepAlive;
                        }
                    }
                    else if (
                            (lowecaseHeader == "accept-language") ||
                            (lowecaseHeader == "acceptlanguage") ||
                            (lowecaseHeader == "accept language")
                            )

                    {
                        req.Headers["Accept-Language"] = headerValue;
                    }
                    else if (
                            (lowecaseHeader == "user-agent") ||
                            (lowecaseHeader == "useragent") ||
                            (lowecaseHeader == "user agent")
                            )
                    {
                        req.UserAgent = headerValue;
                    }
                    else if (
                            (lowecaseHeader == "content-type") ||
                            (lowecaseHeader == "contenttype") ||
                            (lowecaseHeader == "content type")
                            )
                    {
                        req.ContentType = headerValue;
                    }
                    else
                    {
                        req.Headers[header] = headerValue;
                    }
                }
                else
                {
                    break;
                }
            }
        }

        if (((postDict != null) && (postDict.Count > 0)) || (!string.IsNullOrEmpty(postDataStr)))
        {
            req.Method = "POST";
            if (req.ContentType == null)
            {
                req.ContentType = "application/x-www-form-urlencoded";
            }

            if ((postDict != null) && (postDict.Count > 0))
            {
                postDataStr = _quoteParas(postDict);
            }
                        
            //byte[] postBytes = Encoding.GetEncoding("utf-8").GetBytes(postData);
            byte[] postBytes = Encoding.UTF8.GetBytes(postDataStr);
            req.ContentLength = postBytes.Length;

            try
            {
                Stream postDataStream = req.GetRequestStream();
                postDataStream.Write(postBytes, 0, postBytes.Length);
                postDataStream.Close();
            }
            catch (WebException webEx)
            {
                //for prev has set ReadWriteTimeout
                //so here also may timeout
                if (webEx.Status == WebExceptionStatus.Timeout)
                {
                    req = null;
                }
            }
        }
        else
        {
            req.Method = "GET";
        }

        if (req != null)
        {
            //may timeout, has fixed in:
            //http://www.crifan.com/fixed_problem_sometime_httpwebrequest_getresponse_timeout/
            try
            {
                resp = (HttpWebResponse)req.GetResponse();
                updateLocalCookies(resp.Cookies, ref curCookies);
            }
            catch (WebException webEx)
            {
                if (webEx.Status == WebExceptionStatus.Timeout)
                {
                    resp = null;
                }
            }
        }
        
        return resp;
    }

#if USE_GETURLRESPONSE_BW
    private void getUrlResponse_bw(string url,
                                    Dictionary<string, string> headerDict = defHeaderDict,
                                    Dictionary<string, string> postDict = defPostDict,
                                    int timeout = defTimeout,
                                    string postDataStr = defPostDataStr,
                                    int readWriteTimeout = defReadWriteTimeout)
    {
        // Create a background thread
        BackgroundWorker bgwGetUrlResp = new BackgroundWorker();
        bgwGetUrlResp.DoWork += new DoWorkEventHandler(bgwGetUrlResp_DoWork);
        bgwGetUrlResp.RunWorkerCompleted += new RunWorkerCompletedEventHandler( bgwGetUrlResp_RunWorkerCompleted );

        //init
        bNotCompleted_resp = true;
            
        // run in another thread
        object paraObj = new object[] { url, headerDict, postDict, timeout, postDataStr, readWriteTimeout };
        bgwGetUrlResp.RunWorkerAsync(paraObj);
    }

    private void bgwGetUrlResp_DoWork(object sender, DoWorkEventArgs e)
    {
        object[] paraObj = (object[])e.Argument;
        string url = (string)paraObj[0];
        Dictionary<string, string> headerDict = (Dictionary<string, string>)paraObj[1];
        Dictionary<string, string> postDict = (Dictionary<string, string>)paraObj[2];
        int timeout = (int)paraObj[3];
        string postDataStr = (string)paraObj[4];
        int readWriteTimeout = (int)paraObj[5];

        e.Result = _getUrlResponse(url, headerDict, postDict, timeout, postDataStr, readWriteTimeout);
    }

    //void m_bgWorker_ProgressChanged(object sender, ProgressChangedEventArgs e)
    //{
    //    bRespNotCompleted = true;
    //}

    private void bgwGetUrlResp_RunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e)
    {
        // The background process is complete. We need to inspect
        // our response to see if an error occurred, a cancel was
        // requested or if we completed successfully.

        // Check to see if an error occurred in the
        // background process.
        if (e.Error != null)
        {
            //MessageBox.Show(e.Error.Message);
            return;
        }

        // Check to see if the background process was cancelled.
        if (e.Cancelled)
        {
            //MessageBox.Show("Cancelled ...");
        }
        else
        {
            bNotCompleted_resp = false;

            // Everything completed normally.
            // process the response using e.Result
            //MessageBox.Show("Completed...");
            gCurResp = (HttpWebResponse)e.Result;
        }
    }
#endif


    private void getUrlRespStreamBytes_bw(ref Byte[] respBytesBuf,
                                string url,
                                Dictionary<string, string> headerDict,
                                Dictionary<string, string> postDict,
                                int timeout,
                                Action<int> funcUpdateProgress)
    {
        // Create a background thread
        gBgwDownload = new BackgroundWorker();
        gBgwDownload.DoWork += bgwDownload_DoWork;
        gBgwDownload.RunWorkerCompleted += bgwDownload_RunWorkerCompleted;
        gBgwDownload.WorkerReportsProgress = true;
        gBgwDownload.ProgressChanged += bgwDownload_ProgressChanged;

        //init
        bNotCompleted_download = true;
        gFuncUpdateProgress = funcUpdateProgress;
        
        // run in another thread
        object paraObj = new object[] {respBytesBuf, url, headerDict, postDict, timeout};
        gBgwDownload.RunWorkerAsync(paraObj);
    }

    private void bgwDownload_ProgressChanged(object sender, ProgressChangedEventArgs e)
    {
        if (gFuncUpdateProgress != null)
        {
            // This function fires on the UI thread so it's safe to edit
            // the UI control directly, no funny business with Control.Invoke.
            // Update the progressBar with the integer supplied to us from the
            // ReportProgress() function.  Note, e.UserState is a "tag" property
            // that can be used to send other information from the
            // BackgroundThread to the UI thread.

            gFuncUpdateProgress(e.ProgressPercentage);
        }
    }

    private void bgwDownload_DoWork(object sender, DoWorkEventArgs e)
    {
    //    // The sender is the BackgroundWorker object we need it to
    //    // report progress and check for cancellation.
    //    BackgroundWorker gBgwDownload = sender as BackgroundWorker;

        object[] paraObj = (object[])e.Argument;
        Byte[] respBytesBuf = (Byte[])paraObj[0];
        string url = (string)paraObj[1];
        Dictionary<string, string> headerDict = (Dictionary<string, string>)paraObj[2];
        Dictionary<string, string> postDict = (Dictionary<string, string>)paraObj[3];
        int timeout = (int)paraObj[4];

        //e.Result = _getUrlRespStreamBytes(ref respBytesBuf, url, headerDict, postDict, timeout);
        

        int curReadoutLen;
        int realReadoutLen = 0;
        int curBufPos = 0;
        
        long totalLength = 0;
        long currentLength = 0;

        try
        {
            //HttpWebResponse resp = getUrlResponse(url, headerDict, postDict, timeout);
            HttpWebResponse resp = getUrlResponse(url, headerDict, postDict);
            long expectReadoutLen = resp.ContentLength;

            totalLength = expectReadoutLen;
            currentLength = 0;

            Stream binStream = resp.GetResponseStream();
            //int streamDataLen  = (int)binStream.Length; // erro: not support seek operation

            do
            {
                //let up layer update its UI, otherwise up layer UI will no response during this func exec time
                //now has make this function to call by backgroundworker, so not need this to update UI
                //System.Windows.Forms.Application.DoEvents();

                // here download logic is:
                // once request, return some data
                // request multiple time, until no more data
                curReadoutLen = binStream.Read(respBytesBuf, curBufPos, (int)expectReadoutLen);
                if (curReadoutLen > 0)
                {
                    curBufPos += curReadoutLen;

                    currentLength = curBufPos;

                    expectReadoutLen = expectReadoutLen - curReadoutLen;

                    realReadoutLen += curReadoutLen;

                    int currentPercent = (int)((currentLength * 100) / totalLength);
                    
                    if (currentPercent < 0)
                    {
                        currentPercent = 0;
                    }

                    if (currentPercent > 100)
                    {
                        currentPercent = 100;
                    }

                    gBgwDownload.ReportProgress(currentPercent);
                }
            } while (curReadoutLen > 0);
        }
        catch (Exception ex)
        {
            string errorMessage = ex.Message;
            realReadoutLen = -1;
        }

        //return realReadoutLen;
        
        e.Result = realReadoutLen;
        //gBgwDownload.ReportProgress(100);
    }

    private void bgwDownload_RunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e)
    {
        // The background process is complete. We need to inspect
        // our response to see if an error occurred, a cancel was
        // requested or if we completed successfully.

        // Check to see if an error occurred in the
        // background process.
        if (e.Error != null)
        {
            //MessageBox.Show(e.Error.Message);
            return;
        }

        // Check to see if the background process was cancelled.
        if (e.Cancelled)
        {
            //MessageBox.Show("Cancelled ...");
        }
        else
        {
            bNotCompleted_download = false;

            // Everything completed normally.
            // process the response using e.Result
            //MessageBox.Show("Completed...");
            gRealReadoutLen = (int)e.Result;
        }
    }