mysqlHelper使用using释放资源。线程处理bug修复，最后的时候线程数可能小于预定数。

0dayso · Jun 9, 2017 · eb06119 · eb06119
1 parent 86222ec
commit eb06119
Show file tree

Hide file tree

Showing 78 changed files with 108,670 additions and 67 deletions.
diff --git a/.vs/ZJCredit/v14/.suo b/.vs/ZJCredit/v14/.suo
diff --git a/AdministrativePenalty/AdministrativePenaltyCrawler.cs b/AdministrativePenalty/AdministrativePenaltyCrawler.cs
@@ -33,6 +33,8 @@ public AdministrativePenaltyCrawler()
             _interval = 80;
         }
 
+
+
         public void Run()
         {
             var httpHelper = new HttpHelper { Timeout = 5 * 60 * 1000 };
@@ -42,11 +44,11 @@ public void Run()
             InitHttpEncoding(url, httpHelper);
             InitTotalRecord(url, httpHelper);
             InitUrlQueue();
-            var threadNum = 5;
-            var tasks = new Task[threadNum];
 
             while (_urlQueue.Count!=0)
             {
+                var threadNum = _urlQueue.Count>5?5:_urlQueue.Count;
+                var tasks = new Task[threadNum];
                 for (var i = 0; i < threadNum; i++)
                 {
                     if (_urlQueue.Count == 0)
@@ -68,8 +70,6 @@ public void Run()
 
 
 
-
-
         private void GetInfoAndInsertDb(object o)
         {
             try
@@ -208,9 +208,11 @@ private void Test()
         {
             var httpHelper = new HttpHelper();
             //httpHelper.HttpEncoding = HttpHelper.GetHtmlEncoding(httpHelper.GetHtmlByGet("https://github.com/charles427"));
-            var html = httpHelper.GetHtmlByGet("https://github.com/charles427", new WebProxy("91.121.165.21:443"));
+            var html = httpHelper.GetHtmlByGet("https://www.bing.com/", new WebProxy("89.188.229.14:8080"));
             //var html = httpHelper.GetHtmlByGet("http://dev.kuaidaili.com/api/getproxy/?orderid=909664274113782&num=100&b_pcchrome=1&b_pcie=1&b_pcff=1&protocol=1&method=2&an_an=1&an_ha=1&sep=1");
+            Console.WriteLine(html);
 
+            Console.WriteLine("123");
         }
     }
 }

diff --git a/AdministrativePenalty/bin/Debug/AdministrativePenalty.exe b/AdministrativePenalty/bin/Debug/AdministrativePenalty.exe
diff --git a/AdministrativePenalty/bin/Debug/AdministrativePenalty.pdb b/AdministrativePenalty/bin/Debug/AdministrativePenalty.pdb
diff --git a/AdministrativePenalty/bin/Debug/HelpLib.dll b/AdministrativePenalty/bin/Debug/HelpLib.dll
diff --git a/AdministrativePenalty/bin/Debug/HelpLib.pdb b/AdministrativePenalty/bin/Debug/HelpLib.pdb
diff --git a/AdministrativePenalty/bin/Debug/Newtonsoft.Json.dll b/AdministrativePenalty/bin/Debug/Newtonsoft.Json.dll
diff --git a/AdministrativePenalty/bin/Debug/Newtonsoft.Json.xml b/AdministrativePenalty/bin/Debug/Newtonsoft.Json.xml
diff --git a/AdministrativePenalty/bin/Release/AdministrativePenalty.exe b/AdministrativePenalty/bin/Release/AdministrativePenalty.exe
diff --git a/AdministrativePenalty/bin/Release/AdministrativePenalty.pdb b/AdministrativePenalty/bin/Release/AdministrativePenalty.pdb
diff --git a/AdministrativePenalty/bin/Release/HelpLib.dll b/AdministrativePenalty/bin/Release/HelpLib.dll
diff --git a/AdministrativePenalty/bin/Release/HelpLib.pdb b/AdministrativePenalty/bin/Release/HelpLib.pdb
diff --git a/AdministrativePenalty/bin/Release/Newtonsoft.Json.dll b/AdministrativePenalty/bin/Release/Newtonsoft.Json.dll
diff --git a/AdministrativePenalty/bin/Release/Newtonsoft.Json.xml b/AdministrativePenalty/bin/Release/Newtonsoft.Json.xml
diff --git a/AdministrativePenalty/obj/Debug/AdministrativePenalty.csproj.FileListAbsolute.txt b/AdministrativePenalty/obj/Debug/AdministrativePenalty.csproj.FileListAbsolute.txt
@@ -13,3 +13,5 @@ F:\myGitHub\ZJCredit\AdministrativePenalty\obj\Debug\AdministrativePenalty.pdb
 F:\myGitHub\ZJCredit\AdministrativePenalty\bin\Debug\log4net.config
 F:\myGitHub\ZJCredit\AdministrativePenalty\bin\Debug\log4net.dll
 F:\myGitHub\ZJCredit\AdministrativePenalty\bin\Debug\log4net.xml
+F:\myGitHub\ZJCredit\AdministrativePenalty\bin\Debug\Newtonsoft.Json.dll
+F:\myGitHub\ZJCredit\AdministrativePenalty\bin\Debug\Newtonsoft.Json.xml
diff --git a/AdministrativePenalty/obj/Debug/AdministrativePenalty.csprojResolveAssemblyReference.cache b/AdministrativePenalty/obj/Debug/AdministrativePenalty.csprojResolveAssemblyReference.cache
diff --git a/AdministrativePenalty/obj/Debug/AdministrativePenalty.exe b/AdministrativePenalty/obj/Debug/AdministrativePenalty.exe
diff --git a/AdministrativePenalty/obj/Debug/AdministrativePenalty.pdb b/AdministrativePenalty/obj/Debug/AdministrativePenalty.pdb
diff --git a/AdministrativePenalty/obj/Release/AdministrativePenalty.csproj.FileListAbsolute.txt b/AdministrativePenalty/obj/Release/AdministrativePenalty.csproj.FileListAbsolute.txt
@@ -13,3 +13,5 @@ F:\myGitHub\ZJCredit\AdministrativePenalty\obj\Release\AdministrativePenalty.pdb
 F:\myGitHub\ZJCredit\AdministrativePenalty\bin\Release\log4net.dll
 F:\myGitHub\ZJCredit\AdministrativePenalty\bin\Release\log4net.xml
 F:\myGitHub\ZJCredit\AdministrativePenalty\bin\Release\log4net.config
+F:\myGitHub\ZJCredit\AdministrativePenalty\bin\Release\Newtonsoft.Json.dll
+F:\myGitHub\ZJCredit\AdministrativePenalty\bin\Release\Newtonsoft.Json.xml
diff --git a/AdministrativePenalty/obj/Release/AdministrativePenalty.csprojResolveAssemblyReference.cache b/AdministrativePenalty/obj/Release/AdministrativePenalty.csprojResolveAssemblyReference.cache
diff --git a/AdministrativePenalty/obj/Release/AdministrativePenalty.exe b/AdministrativePenalty/obj/Release/AdministrativePenalty.exe
diff --git a/AdministrativePenalty/obj/Release/AdministrativePenalty.pdb b/AdministrativePenalty/obj/Release/AdministrativePenalty.pdb
diff --git a/HMD/HMDCrawler.cs b/HMD/HMDCrawler.cs
@@ -42,8 +42,9 @@ public void Run()
 
             while (_urlQueue.Count!=0)
             {
-                var threadTotalNum = 5;
+                var threadTotalNum = _urlQueue.Count>5?5:_urlQueue.Count;
                 var taskArray = new Task[threadTotalNum];
+
                 for (var i = 0; i < threadTotalNum; i++)
                 {
                     //判断队列是否已经取完 若取完则退出循环
@@ -55,9 +56,9 @@ public void Run()
                 }
 
                 //等待这几个线程结束
-                for (var i = 0; i < threadTotalNum; i++)
+                for (var j = 0; j < threadTotalNum; j++)
                 {
-                    taskArray[i].Wait();
+                    taskArray[j].Wait();
                 }
 
             }

diff --git a/HMD/bin/Debug/HelpLib.dll b/HMD/bin/Debug/HelpLib.dll
diff --git a/HMD/bin/Debug/HelpLib.pdb b/HMD/bin/Debug/HelpLib.pdb
diff --git a/HMD/bin/Debug/Newtonsoft.Json.dll b/HMD/bin/Debug/Newtonsoft.Json.dll
diff --git a/HMD/bin/Debug/Newtonsoft.Json.xml b/HMD/bin/Debug/Newtonsoft.Json.xml
diff --git a/HMD/bin/Debug/ZJCredit.exe b/HMD/bin/Debug/ZJCredit.exe
diff --git a/HMD/bin/Debug/ZJCredit.pdb b/HMD/bin/Debug/ZJCredit.pdb
diff --git a/HMD/bin/Release/HelpLib.dll b/HMD/bin/Release/HelpLib.dll
diff --git a/HMD/bin/Release/HelpLib.pdb b/HMD/bin/Release/HelpLib.pdb
diff --git a/HMD/bin/Release/Newtonsoft.Json.dll b/HMD/bin/Release/Newtonsoft.Json.dll
diff --git a/HMD/bin/Release/Newtonsoft.Json.xml b/HMD/bin/Release/Newtonsoft.Json.xml
diff --git a/HMD/bin/Release/ZJCredit.exe b/HMD/bin/Release/ZJCredit.exe
diff --git a/HMD/bin/Release/ZJCredit.pdb b/HMD/bin/Release/ZJCredit.pdb
diff --git a/HMD/obj/Debug/HMD.csproj.FileListAbsolute.txt b/HMD/obj/Debug/HMD.csproj.FileListAbsolute.txt
@@ -13,3 +13,5 @@ F:\myGitHub\ZJCredit\HMD\obj\Debug\HMD.csprojResolveAssemblyReference.cache
 F:\myGitHub\ZJCredit\HMD\bin\Debug\log4net.config
 F:\myGitHub\ZJCredit\HMD\bin\Debug\log4net.dll
 F:\myGitHub\ZJCredit\HMD\bin\Debug\log4net.xml
+F:\myGitHub\ZJCredit\HMD\bin\Debug\Newtonsoft.Json.dll
+F:\myGitHub\ZJCredit\HMD\bin\Debug\Newtonsoft.Json.xml
diff --git a/HMD/obj/Debug/HMD.csprojResolveAssemblyReference.cache b/HMD/obj/Debug/HMD.csprojResolveAssemblyReference.cache
diff --git a/HMD/obj/Debug/ZJCredit.exe b/HMD/obj/Debug/ZJCredit.exe
diff --git a/HMD/obj/Debug/ZJCredit.pdb b/HMD/obj/Debug/ZJCredit.pdb
diff --git a/HMD/obj/Release/HMD.csproj.FileListAbsolute.txt b/HMD/obj/Release/HMD.csproj.FileListAbsolute.txt
@@ -25,3 +25,5 @@ F:\myGitHub\ZJCredit\HMD\obj\Release\HMD.csprojResolveAssemblyReference.cache
 F:\myGitHub\ZJCredit\HMD\bin\Release\log4net.dll
 F:\myGitHub\ZJCredit\HMD\bin\Release\log4net.xml
 F:\myGitHub\ZJCredit\HMD\bin\Release\log4net.config
+F:\myGitHub\ZJCredit\HMD\bin\Release\Newtonsoft.Json.dll
+F:\myGitHub\ZJCredit\HMD\bin\Release\Newtonsoft.Json.xml
diff --git a/HMD/obj/Release/HMD.csprojResolveAssemblyReference.cache b/HMD/obj/Release/HMD.csprojResolveAssemblyReference.cache
diff --git a/HMD/obj/Release/ZJCredit.exe b/HMD/obj/Release/ZJCredit.exe
diff --git a/HMD/obj/Release/ZJCredit.pdb b/HMD/obj/Release/ZJCredit.pdb
diff --git a/HelpLib/HelpLib.csproj b/HelpLib/HelpLib.csproj
@@ -48,6 +48,10 @@
       <HintPath>..\packages\MySql.Data.6.9.8\lib\net40\MySql.Data.dll</HintPath>
       <Private>True</Private>
     </Reference>
+    <Reference Include="Newtonsoft.Json, Version=8.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
+      <HintPath>..\packages\Newtonsoft.Json.8.0.3\lib\net40\Newtonsoft.Json.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
     <Reference Include="System" />
     <Reference Include="System.Data" />
     <Reference Include="System.Xml" />
@@ -58,6 +62,7 @@
     <Compile Include="Log4NetHelper.cs" />
     <Compile Include="MySqlHelper.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="ProxyHelper.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="app.config" />

diff --git a/HelpLib/MySqlHelper.cs b/HelpLib/MySqlHelper.cs
@@ -143,16 +143,15 @@ public Dictionary<string, List<object>> GetSelectDic(MySqlCommand mySqlCommand)
         /// <returns></returns>
         public Dictionary<string, List<object>> GetSelectDicBySql(string sqlSelect)
         {
-            var mySqlConnection = GetMySqlConnection();
-            var mySqlCommand = GetMySqlCommand(sqlSelect, mySqlConnection);
-
-            mySqlConnection.Open();
-
-            var dic = GetSelectDic(mySqlCommand);
-
-            mySqlConnection.Close();
-
-            return dic;
+            using (var mySqlConnection = GetMySqlConnection())
+            {
+                using (var mySqlCommand = GetMySqlCommand(sqlSelect, mySqlConnection))
+                {
+                    mySqlConnection.Open();
+                    var dic = GetSelectDic(mySqlCommand);
+                    return dic;
+                }
+            }
         }
 
         /// <summary>
@@ -170,11 +169,14 @@ public void UpdateTable(MySqlCommand mySqlCommand)
         /// <param name="sqlUpdate"></param>
         public void UpdateTable(string sqlUpdate)
         {
-            var mySqlConnection = GetMySqlConnection();
-            var mySqlCommand = GetMySqlCommand(sqlUpdate, mySqlConnection);
-            mySqlConnection.Open();
-            UpdateTable(mySqlCommand);
-            mySqlConnection.Close();
+            using (var mySqlConnection = GetMySqlConnection())
+            {
+                using (var mySqlCommand = GetMySqlCommand(sqlUpdate, mySqlConnection))
+                {
+                    mySqlConnection.Open();
+                    UpdateTable(mySqlCommand);
+                }
+            }
         }
 
         /// <summary>
@@ -192,11 +194,14 @@ public void DeleteTable(MySqlCommand mySqlCommand)
         /// <param name="sqlDelete"></param>
         public void DeleteTable(string sqlDelete)
         {
-            var mySqlConnection = GetMySqlConnection();
-            var mySqlCommand = GetMySqlCommand(sqlDelete, mySqlConnection);
-            mySqlConnection.Open();
-            DeleteTable(mySqlCommand);
-            mySqlConnection.Close();
+            using (var mySqlConnection = GetMySqlConnection())
+            {
+                using (var mySqlCommand = GetMySqlCommand(sqlDelete, mySqlConnection))
+                {
+                    mySqlConnection.Open();
+                    DeleteTable(mySqlCommand);
+                }
+            }
         }
 
         /// <summary>
@@ -214,11 +219,14 @@ public void InsertTable(MySqlCommand mySqlCommand)
         /// <param name="sqlInsert"></param>
         public void InsertTable(string sqlInsert)
         {
-            var mySqlConnection = GetMySqlConnection();
-            var mySqlCommand = GetMySqlCommand(sqlInsert, mySqlConnection);
-            mySqlConnection.Open();
-            InsertTable(mySqlCommand);
-            mySqlConnection.Close();
+            using (var mySqlConnection = GetMySqlConnection())
+            {
+                using (var mySqlCommand = GetMySqlCommand(sqlInsert, mySqlConnection))
+                {
+                    mySqlConnection.Open();
+                    InsertTable(mySqlCommand);
+                }
+            }
         }
 
 
@@ -262,26 +270,26 @@ public void InsertTable(Dictionary<string, string> infoDic, string tableName)
 
                 var str = $"{strPart1}{strPart2}";
 
-                var mySqlCommand = GetMySqlCommand(str, mySqlConnection);
+                using (var mySqlCommand = GetMySqlCommand(str, mySqlConnection))
+                {
 
 
-                foreach (var info in infoDic)
-                {
-                    mySqlCommand.Parameters.AddWithValue($"@{info.Key}", info.Value);
-                }
+                    foreach (var info in infoDic)
+                    {
+                        mySqlCommand.Parameters.AddWithValue($"@{info.Key}", info.Value);
+                    }
 
-                switch (mySqlCommand.ExecuteNonQuery())
-                {
-                    case 1:
-                        Console.WriteLine($"向{tableName}表插入新记录成功！");
-                        break;
-                    case 0:
-                        throw new Exception($"向{tableName}表插入新记录失败！");
-                    default:
-                        throw new Exception($"向{tableName}表插入了多条新记录！");
+                    switch (mySqlCommand.ExecuteNonQuery())
+                    {
+                        case 1:
+                            Console.WriteLine($"向{tableName}表插入新记录成功！");
+                            break;
+                        case 0:
+                            throw new Exception($"向{tableName}表插入新记录失败！");
+                        default:
+                            throw new Exception($"向{tableName}表插入了多条新记录！");
+                    }
                 }
-                //关闭连接
-                mySqlConnection.Close();
             }
 
         }
@@ -307,11 +315,11 @@ public void CreateTable(Dictionary<string, string> infoDic, Dictionary<string, s
         public bool GetLock(MySqlConnection mySqlConnection, string lockName, int timeout = 10)
         {
             var sqlLock = $"SELECT GET_LOCK('{lockName}',{timeout});";
-            var mySqlCommandLock = GetMySqlCommand(sqlLock, mySqlConnection);
-            //mySqlConnection.Open();
-            var result = mySqlCommandLock.ExecuteScalar();
-            //mySqlConnection.Close();
-            return result.ToString() == "1";
+            using (var mySqlCommandLock = GetMySqlCommand(sqlLock, mySqlConnection))
+            {
+                var result = mySqlCommandLock.ExecuteScalar();
+                return result.ToString() == "1";
+            }
         }
 
 
@@ -324,11 +332,11 @@ public bool GetLock(MySqlConnection mySqlConnection, string lockName, int timeou
         public bool GetReleaseLock(MySqlConnection mySqlConnection, string lockName)
         {
             var sqlUnLock = $"SELECT RELEASE_LOCK('{lockName}');";
-            var mySqlCommandReleaseLock = GetMySqlCommand(sqlUnLock, mySqlConnection);
-            //mySqlConnection.Open();
-            var result = mySqlCommandReleaseLock.ExecuteScalar();
-            //mySqlConnection.Close();
-            return result.ToString() == "1";
+            using (var mySqlCommandReleaseLock = GetMySqlCommand(sqlUnLock, mySqlConnection))
+            {
+                var result = mySqlCommandReleaseLock.ExecuteScalar();
+                return result.ToString() == "1";
+            }
         }
 
 
@@ -339,16 +347,17 @@ public void Test()
         {
             //var mysqlHelper = new MysqlHelper("127.0.0.1", "zhanxian", "root", "root");
             var mySqlHelper = new MySqlHelper();
-            var mySqlConnection = GetMySqlConnection();
-            var sqlSelect = "SELECT company FROM jd";
-            var mySqlCommand = GetMySqlCommand(sqlSelect, mySqlConnection);
-
-            mySqlConnection.Open();
-
-            var dic = GetSelectDic(mySqlCommand);
+            using (var mySqlConnection = GetMySqlConnection())
+            {
+                var sqlSelect = "SELECT company FROM jd";
+                using (var mySqlCommand = GetMySqlCommand(sqlSelect, mySqlConnection))
+                {
 
-            mySqlConnection.Close();
+                    mySqlConnection.Open();
 
+                    var dic = GetSelectDic(mySqlCommand);
+                }
+            }
         }
 
         /// <summary>