diff --git a/KsatSupplymentLibrary/Model/Uploader/UploadCache.cs b/KsatSupplymentLibrary/Model/Uploader/UploadCache.cs index e705431..dd2d35f 100644 --- a/KsatSupplymentLibrary/Model/Uploader/UploadCache.cs +++ b/KsatSupplymentLibrary/Model/Uploader/UploadCache.cs @@ -18,6 +18,8 @@ namespace Ksat.Supplyment.Library.Model.Uploader [Required] public string UploaderID { get; set; } + public string Tag { get; set; } + [Required] public DateTime CreateAt { get; set; } diff --git a/KsatSupplymentLibrary/Model/Uploader/UploadCancel.cs b/KsatSupplymentLibrary/Model/Uploader/UploadCancel.cs index 683587c..d2e2351 100644 --- a/KsatSupplymentLibrary/Model/Uploader/UploadCancel.cs +++ b/KsatSupplymentLibrary/Model/Uploader/UploadCancel.cs @@ -18,6 +18,8 @@ namespace Ksat.Supplyment.Library.Model.Uploader [Required] public string UploaderID { get; set; } + public string Tag { get; set; } + [Required] public DateTime CreateAt { get; set; } diff --git a/KsatSupplymentLibrary/Model/Uploader/UploadFinish.cs b/KsatSupplymentLibrary/Model/Uploader/UploadFinish.cs index 0425418..e324d67 100644 --- a/KsatSupplymentLibrary/Model/Uploader/UploadFinish.cs +++ b/KsatSupplymentLibrary/Model/Uploader/UploadFinish.cs @@ -18,6 +18,8 @@ namespace Ksat.Supplyment.Library.Model.Uploader [Required] public string UploaderID { get; set; } + public string Tag { get; set; } + [Required] public DateTime CreateAt { get; set; } diff --git a/KsatSupplymentLibrary/Model/Uploader/UploadModel.cs b/KsatSupplymentLibrary/Model/Uploader/UploadModel.cs index 3128fea..828ec6b 100644 --- a/KsatSupplymentLibrary/Model/Uploader/UploadModel.cs +++ b/KsatSupplymentLibrary/Model/Uploader/UploadModel.cs @@ -25,6 +25,8 @@ namespace Ksat.Supplyment.Library.Model.Uploader /// public string ErrorInfo; + public string Tag { get; set; } + public T Request; } } diff --git a/KsatSupplymentLibrary/Uploader/Uploader.cs b/KsatSupplymentLibrary/Uploader/Uploader.cs index ddbb11c..f20047a 100644 --- a/KsatSupplymentLibrary/Uploader/Uploader.cs +++ b/KsatSupplymentLibrary/Uploader/Uploader.cs @@ -2,7 +2,6 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading; using Ksat.Supplyment.Library.Model.Uploader; using Ksat.Supplyment.Library.Model; @@ -283,24 +282,32 @@ namespace Ksat.Supplyment.Library.Uploader } } - private void cacheRequest(List datas) + private void cacheRequest(List> datas) { - using (var db = new CodeFirstDbContext()) + try { - //sLog.Formf($"cache requests,{datas.Count()}"); - var caches = datas.Select((data) => - new UploadCache + using (var db = new CodeFirstDbContext()) { - UploaderID = mTaskIdentify, - CreateAt = DateTime.Now, - RetryAt = new DateTime(1970, 1, 1), - RetryCount = 0, - ErrorInfo = "", - RequestData = mOperator.ConvertRequestToCachel(data) + //sLog.Formf($@"cache requests,{datas.Count()}"); + var caches = datas.Select((data) => + new UploadCache + { + UploaderID = mTaskIdentify, + CreateAt = DateTime.Now, + RetryAt = new DateTime(1970, 1, 1), + RetryCount = 0, + ErrorInfo = "", + Tag = data.Item2, + RequestData = mOperator.ConvertRequestToCachel(data.Item1) + } + ); + db.UploadCaches.AddRange(caches); + db.SaveChanges(); } - ); - db.UploadCaches.AddRange(caches); - db.SaveChanges(); + } + catch (Exception ex) + { + //sLog.FormErrorf(ex.Message); } } @@ -378,7 +385,7 @@ namespace Ksat.Supplyment.Library.Uploader /// /// 添加上传请求 /// - public void AddUploadRequest(T data) + public void AddUploadRequest(T data, string tag) { if (!Interlocked.Equals(mIsCanceled, 1)) { @@ -389,6 +396,7 @@ namespace Ksat.Supplyment.Library.Uploader RetryAt = new DateTime(1970, 1, 1), RetryCount = 0, ErrorInfo = "", + Tag = tag, Request = data }; if (mDatas.Count >= MaxOperatingQueueSize) @@ -405,7 +413,7 @@ namespace Ksat.Supplyment.Library.Uploader else { //sLog.Formf("task canceled"); - cacheRequest(new List() { data }); + cacheRequest(new List>() { new Tuple(data, tag) }); } } } diff --git a/KsatSupplymentLibrary/flow.drawio b/KsatSupplymentLibrary/flow.drawio index 2616b23..5e77b77 100644 --- a/KsatSupplymentLibrary/flow.drawio +++ b/KsatSupplymentLibrary/flow.drawio @@ -1 +1 @@ -5VzbcqM4EP0aHrOFkATiEdt4tmr2MjV52NmnLWIrNrtgORjH9nz9SkKYm5wh4wTZTtWUB1oIwVH3Uau7iQXH6f5TFq2Xv7M5TSzHnu8tOLEcB2AM+H9CclASUEoWWTxXskpwH3+nSmgr6Tae003jwpyxJI/XTeGMrVZ0ljdkUZaxXfOyR5Y0R11HC9oR3M+ipCv9K57ny0JKHK+S/0rjxbIcGbh+0ZJG5cXqTTbLaM52NREMLTjOGMuLo3Q/polAr8Sl6Dc90Xp8sIyu8j4d9p8/fU3/+YxX0bf7pzD6cvc13NzB4i7PUbJVL6weNj+UCNA5B0SdsixfsgVbRUlYSUcZ267mVAxj87Pqmt8YW3Mh4MJ/aZ4f1OxG25xx0TJPE9VajCkGOvluSrRh22xGX3ihUkeibEHzF65zjjPAdZeylObZgffLaBLl8XPzOSKlQ4vjdRXM/EAh/QrUQRf10LVIaAVYHPjIGrlWyH9H4l+ILRJYAejMTIW7AHG3jHN6v44kPDtujjqMn2mW0/3LKHdRUR0colS5NGakzneVZYBS3Zc1q3DtdwISapC8bv11euovMqm/jlZ/R15NW22pv0Qci4NxKblsjT6eG9No99YUGvVUaGxSoaFGoz8G7J5J2JGWR3xiBVPBEb5tBa44EMzCmyS/ENnEf31Xkg5fN4kVTgXF+NMuoSxZ+rDdDEIm0G6SCdSQCdFwCXkvLsEaeDkFc+gmAsNgIqAWYBJrZKeTKI+MEzJCLQx904QMb85F9npSg2+SGjy97rYcCl9osD9VSuzzTm6UCq1c8BcbgdIgL4QQkEaZByUEouXbtt+GBf2Oxpsdh+TS+ADbpvnA12BIJKmOdPp4YQ6u65nGD2oW/evm0zJU9UNCBSfmaqCgg90BfrtOWDTnsqct5TLTyur6FxdfuLntGOgbIINmN2Sa9f+DAA+gUZbQxSavZQfmgVY4xzXscIHb4w/YV42NRiihxtX9IMAXb2qMPzTbZcEfE8uHkjZ8i0w0kWAR9+GX2c0dHL8BuihGMR7TAfja9bhvZPJsb1l2DbIsOtQuWLN4lW9qd/4iBNWE++0Jt+3WnBV3rGbw+GhnTKouEOqJAB03F2E9oUUwsNNN1xYGdtGJ14LHeHwOdD3FVP5no7uVcbzaWxpkPMGEutvAa2cU0pNRoNGIJtBF3+Q6SMIi+F4EOINJzb2+uIwpbgeUjBMA6noI16W/Tt/40dme3U+tiKhF+S4YYEV0dCzlJsInfGTy6WYsYZlscZ+2TDqLDvRo5FK7LnILP7JIGah0lwzOlqYnw9xHg1Nj8Gcuhim7y90tGZftD1njzuI+lQcrSnkmVhDKlTuQfm/NsGVsXQ7oiVaNJ1yMRdRDBUEp8Y0bP2olNHzj0XjYzfBcmfH3LlgzYvzAdvVTPu3bYRi6OBG14lZTJLQ8i4x0q6md3t0ZNyvSMiti3KwcI7tMDld2+Cb6/wIJLAV/S4HN/XwlmOzVEMXZoX72hWYxh4BmSviWlur0tNTC0TPlZpaP2dgPNYIqHJfLCtNiTVJy0KAK1FXK+II7CvpQS7XciIv12JO1ul0Q+VDxekOHccRhE0THeGYXdTXvyhdn6Pc0eWQ0s4tuLsvQG/hTRjJQlkFXDOKKug9ttbNy6GvpM5Ms7La8Ds90dRK6uUo71Nu5t/VTNRB/XHtOoTfOZyclf2oT5baK2Hzy8h7Khy9e/z5bKKTbQrki66+yczIC8dpohwpjygI5oimHM8h3xDTf4ZuLxKO+pcXI6BYJnSgubq3WsthF6LMrCuZ1pS0DxwkAaMVWzBfIYQ1rXLkO980mIaPZJKSv5RZVx7bSYVGHfAlVWQC0v7Izvd9HOr+9myE4XdZ9zAfUv1ISTX6t1+kV8iwieYyTZHxMfsBHMqOzGZdv8oz9R2stDwQjbL/RHLa/ZTD/pSS+uRIv3LfECxutrcPdfVqZRAuQRZBYOblRkO72dsj4GHBaiyXSFINqNfb9HL6bK2rGvb/uNVpNjnX1VeWeRBB/wLVV6m3AVVfGfvnvuR6fGaJ2Wul6bLxAA2uI4Lq0HA/FyydyqqhVRVbmVI/3KJ5Mdaum69XZ3nath4d/kO3tFIc4A4Qq8Ks+bH43R+xNrLXlGnuOaWt1NVEJzUf5RPwSUH2LazqneGnRbLe7M/7jzw5E/IXzJhZREi9W/HjGX1yksUcClngWJYFqSOP5vKBIuom/Rw/yVoIklXXz++KRhSfiXpwVNwVBgs7Cs2Ir+jbYwzJ/WpKApn4BarB3Xo89P63+YlbBJ9UfHoPh/w== \ No newline at end of file +3Vxbc5s6EP41POYMAgnEI74kZ9qeTqZ5OO1Th9iKzSlYDsZJ3F9/JCHMTUlIYyORTsfBKxDi0+6n1e5iy52mT1dZtF3/Q5cksRx7+WS5M8txAMYe+8Mlh0ISOKAQrLJ4KU+qBDfxbyKFtpTu4yXZNU7MKU3yeNsULuhmQxZ5QxZlGX2Up8nu7mjSvOs2WpGO4GYRJV3pv/EyXxdS7PiV/G8Sr9blnYEXFC230eLXKqP7jbzfhm5I0ZJGZTdyULt1tKSPNZE7t9xpRmleHKVPU5JwWEvEiusun2k9Djkjm7zPBU+fr76lPz+jTfT95n4eXV98m+8u3KKXhyjZSyjkYPNDiQ1ZMqjkV5rla7qimyiZV9KJeH7Cb2Ozb9U5XyjdMiFgwv9Inh/kvEf7nDLROk8T2dp9FPl0O7rPFuSF8ZfKEmUrkr9wnnws/iy1G0igrghNSZ4d2AkZSaI8fmiqRSS1a3U8r4KZHUik34A66KI+9yw8t0LEDwJoTTxrzj4n/P8cWTi0QtCZmQp3DuLjOs7JzTYSeD0yO+2J8QPJcvL0Iiqy1cFSlaWVAyi/P1Y2A0p1X9fsxbPPBKSrQHJU+uv01F9olP46Sv2d+DVttYX+Yn7MD6alxGyNdnzdGu2NXKFhT4VGRim0q9DoDwm7bxTsUMkjAbbCS84RgW2FHj/gzMKaBL9g0cQ+A0+QDls3sTW/5BQTXHYJZU3T2/1uEDJx7SaZuEGXTLCCS/C5uAQp4GUUzKCbcQzDGYeag4mtiZ3OopyNQDcjQ9gEEdq6Gdkdu4/s9+SGwChu8NXK2/IoAq7CwaXU4oBd5EUp18oVe9IJKJVHEyM4GDWUGTkKZQZDUgJWMm7bc0OcgCfT3SPDxDRCUGI4KCEECgyxoNWJSiENc3E97S6uq1j2R0WopcW+yqgAGEWp5bhrwO+3CY2WTHa/J3uiXVm9wLgIw9g3ZKUOvqqsrmFbMoUD8DGBB65RwJfjbqxvY9mD+aAV0PE078HA6PnD7avGZsUoXYWr+zGBLx/MEOCBYr/M+WNmBa6gjcDCM0UsmEd+2Gl2cwvHOoBGMYqLdTMKGpke941Nnt5bFpeGWRYdaidsabzJd7Wer7mgmvCgPeG23ZqzosdqBo9De8ekqkKhPg/RMXPh1jO3MAJ2qj9Ah/0WPIoo57AuOuh6iqn4Y8OLjXa82lsaqOCPYfGC3W3gyBgF92QU16yQJlBF38Q6iOci/C4jnOGs5l4blzNF7YCSdgKAXQ/BaP11+saPTu/Z/dGKCFuU74EBVkRHxVJewn3COypGt6AJzUSLd7+nwll0XJ9EHrHrIq/wI4ucgUx4ieBsaXoizH00OHkPNubiNuXlYneLp2X7bdbomfdTebC8mGdmhXOxcofC760Ztoitixv6vFXhCRf3wnJQYVhKAu3GD1s5zkB/eq6b4jHb+HuXrJlh/MD21FN+2feCYejimagVs5oioeVbeKJaTe304kK7WeGWWWHtZuVo2WUyBLPDd379Xy52S8EPIbCZny8Fsyd5i+Lbof7tmmQxg4BkUvgOS3V6Wmrp1xniZpbjbuyHGkEVhotZYVqkSEoOGlRxVbUyAeeOgj7kUi024nw99kW1bhdEdqt4uyPDOOJuE0T9xYuwq3njWpzLDePr9bhmZXbh2LMMvYF3zAK+HHe7hNFX1ztLh76WPtPJwl7L6/B1FyzCsZfawd7OvW2UGsOR5RR643z6pOQfbaK8VhFbgF/eQwXui+efZwsFVVsoj2f9ZXZORCDeGu2QYUxRIIcV5XAa+Q7r5js09kg87FtbDM3aIsFnqotbq7UoduH67PGSeVVpy8BxAgBasRX9BXJo7K/gwb7ZJGhWNgmqa7l51bEtdZjXIZtQlQVAj/fshvU0VX57N0PwfFn3MR9Qf0+JNwW1q55fId9FJHdxkkyPyQ/XtnFkc7PZ5Rn9RRotyBctPJVRk9+Jfyea2/ZrT472BDcae+kX6lv6hcyquUPd/VuZXAuhhSFfUZmx4O62d8i4GXBaiyhUFIkqNfZ8juDYi51R7/d+zaoyR6q6q3KvwheEkGmr0NuQqa6ICbPP93qCZhF42xyQ9oIOpCAIo7UfaePrZ3KwsFV1VuZgj30UQ5WXVdP15uxwuzbER69khzvFJM4AoQ30plehz+a4ncNafe3vQ3qKKIbiNX7MPzGoXt7VnYM0LfrtdXfSXzsIsYfLm1BESbzasOMFg4JnvSccgngRJaFsSOPlsmBIsot/R7eiK86R0rhZv2hioRnvi5HiruBH0FmPxG9bneaXEkDzvWhPocMqFXZOAL29yD99+z6l6df7zZern8HVp99LxQ/a/DAQ+brb0JiMDvKK+ek9GaoC5UEno7uJMNEMzjQZAWxOhqIQ6ESTwb5WPz5XLLTVj/u58/8B \ No newline at end of file diff --git a/S3Demo/Model/DataModel.cs b/S3Demo/Model/DataModel.cs index fc969c2..ae13bc6 100644 --- a/S3Demo/Model/DataModel.cs +++ b/S3Demo/Model/DataModel.cs @@ -8,8 +8,8 @@ namespace S3Demo.Model { public class ReqData { - public string ruleName { get; set; } - public int picReq { get; set; } + public string s3Flag { get; set; } + public int dbFlag { get; set; } } public class RespData { @@ -18,36 +18,8 @@ namespace S3Demo.Model /// public int code { get; set; } /// - /// 错误代码 - /// - public int errorcode { get; set; } - /// - /// 是否有翻页？ - /// - public bool hasPage { get; set; } - /// /// 接口返回message /// public string message { get; set; } - public Result result { get; set; } - } - public class Result - { - /// - /// true 逻辑判断成功 code=200 && successful=true 接口调用成功 - /// - public bool successful { get; set; } - /// - /// Ok/Info Detail - /// - public string message { get; set; } - /// - /// 小板条码 - /// - public List snDetails { get; set; } - /// - /// 绑定信息 - /// - public string emappings { get; set; } } } diff --git a/S3Demo/Run.cs b/S3Demo/Run.cs index 774a6dd..40b10ac 100644 --- a/S3Demo/Run.cs +++ b/S3Demo/Run.cs @@ -1,19 +1,20 @@ -using Amazon; +using System; +using System.IO; +using Newtonsoft.Json; +using Amazon; using Amazon.Runtime; using Amazon.S3; -using System; -using S3Demo.Storage; using S3Demo.Model; -using System.IO; using Ksat.Supplyment.Library.Model; -using Newtonsoft.Json; +using Ksat.Supplyment.Library.Uploader; +using Ksat.Supplyment.Library.Model.Uploader; namespace S3Demo { public class Run { //请求接口 - private static string url = "http://127.0.0.1:9000/api/workflowcenter/anonymous/v1/trackgo/track"; + private static string url = "http://172.17.204.180:9000/api/tri/trackin"; private static BasicAWSCredentials credentials = new BasicAWSCredentials("admin", "admin123."); //private static BasicAWSCredentials credentials = new BasicAWSCredentials("Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); private static string bucketName = "minio/test"; @@ -45,23 +46,9 @@ namespace S3Demo public static void Main() { - /// 写入数据库操作 - //using (var dev = new CodeFirstDbContext()) - //{ - // dev.UploadCaches.Add(new Ksat.Supplyment.Library.Model.Uploader.UploadCache() { - // Id = 1, - // UploaderID = "02", - // CreateAt = DateTime.Now.AddHours(-1), - // RetryAt = DateTime.Now, - // RetryCount = 1, - // ErrorInfo = "null", - // RequestData = "testInfo" - // }); - // var i = dev.SaveChanges(); - // Console.WriteLine(i); - //} try - { + { + #region 目录监控 foreach (var path in paths) { FileSystemWatcher watch = new FileSystemWatcher() @@ -79,8 +66,15 @@ namespace S3Demo // 开始监控 watch.EnableRaisingEvents = true; } + #endregion - #region S3相关测试 + //Uploader uploader = new Uploader("字符串测试", new UploaderOperator()); + //for (var i = 0; i < 20; i++) + //{ + // uploader.AddUploadRequest($"hello{i}"); + //} + + #region S3相关测试 /* //creating Bucket //await CreateBucket.CreatingBucket("minio/bu29", credentials, conf); @@ -126,7 +120,7 @@ namespace S3Demo //await new AmazonS3Client(credentials, conf).DeleteBucketAsync(bucketName); //Console.WriteLine("delete bucket " + bucketName); */ - #endregion + #endregion } catch (Exception ex) { @@ -135,8 +129,68 @@ namespace S3Demo Console.ReadKey(); } + + #region 上传测试 + public class UploaderOperator : IUploaderOperator + { + public string ConvertCacheToRequest(string cache) + { + return cache; + } + + public string ConvertRequestToCachel(string requestData) + { + return requestData; + } + + public bool ShouldRetryRequest(UploadModel requestModel, out string cancelReason) + { + cancelReason = null; + if (requestModel.RetryCount >= 2) + { + cancelReason = "已经两次fail了"; + return false; + } + return true; + } + + public bool ShouldStartRequest(UploadModel requestModel, out string cancelReason) + { + cancelReason = null; + return true; + } + + public IUploaderRequest UploadData(string data) + { + var request = new UploadRequest(); + request.RequestData = data; + return request; + } + } + public class UploadRequest : IUploaderRequest + { + private string mRequestData; + public string RequestData { get => mRequestData; set => mRequestData = value; } + + public event UploadFinishHandler UploadFinish; + + public void Start() + { + Console.WriteLine($"{DateTime.UtcNow.ToString("mm:ss.fff")} task {RequestData} Started"); + System.Threading.Thread.Sleep(100); + var random = new Random(); + + UploadFinish(this, random.Next() % 2 == 0, "task failed"); + } + + public void Cancel() + { + } + } + #endregion + /// - /// 监控到创建文件 + /// 创建事件及主要逻辑 /// /// /// @@ -155,32 +209,34 @@ namespace S3Demo string dt_fomat = DateTime.ParseExact(datetime, "yyyyMMddHHmmssfff", System.Globalization.CultureInfo.CurrentCulture).ToString("yyyy-MM-dd HH:mm:ss:fff"); string code = fileName.Substring(fileName.Length - 6, 2); string sn = fileName.Substring(9, 14); + string now = DateTime.Now.ToString("yyyyMMddHHmmssfff"); String[] tags = new String[] { sn, dt_fomat, code }; string S3Path = lineName + "/" + equipmentName + "/" + date + "/" + fileName; //发送太快会出问题 //没有使用异步线程，可能没有足够的时间等待返回结果 [说明](https://www.cnblogs.com/lnwuyaowei/p/12672866.html) - string flag = ObjectModel.Putobjectswithtags(bucketName, S3Path, e.FullPath, tags, credentials, conf).Result; - if (flag.Equals("OK")) + string s3Flag = ObjectModel.Putobjectswithtags(bucketName, S3Path, e.FullPath, tags, credentials, conf).Result; + if (s3Flag.Equals("OK")) { //保存至数据库 using (var dev = new CodeFirstDbContext()) { - dev.UploadFinishs.Add(new Ksat.Supplyment.Library.Model.Uploader.UploadFinish() + dev.UploadFinishs.Add(new UploadFinish() { UploaderID = lineName, + Tag = sn, CreateAt = DateTime.ParseExact(datetime, "yyyyMMddHHmmssfff", System.Globalization.CultureInfo.CurrentCulture), - RetryAt = DateTime.Now, + RetryAt = DateTime.ParseExact(now, "yyyyMMddHHmmssfff", System.Globalization.CultureInfo.CurrentCulture), RetryCount = 1, RequestData = S3Path }); - int saveInfo = dev.SaveChanges();//1: 保存至数据库成功，否则保存失败 - ReqData req = new ReqData(); - RespData resp = new RespData(); - req.ruleName = "picSave"; - req.picReq = saveInfo; - resp = JsonConvert.DeserializeObject(HttpModel.PostUrl(url, JsonConvert.SerializeObject(req))); - Console.WriteLine("resp: " + resp.code + " " + resp.message); + int dbFlag = dev.SaveChanges();//1: 保存至数据库成功，否则保存失败 + //ReqData req = new ReqData(); + //RespData resp = new RespData(); + //req.s3Flag = s3Flag; + //req.dbFlag = dbFlag; + //resp = JsonConvert.DeserializeObject(HttpModel.PostUrl(url, JsonConvert.SerializeObject(req))); + //Console.WriteLine("resp: " + resp.code + " " + resp.message); } } Console.WriteLine("S3对象存储路径: " + bucketName + "/" + S3Path); diff --git a/S3Demo/S3Demo.csproj b/S3Demo/S3Demo.csproj index a66dd57..d6a8bd7 100644 --- a/S3Demo/S3Demo.csproj +++ b/S3Demo/S3Demo.csproj @@ -79,7 +79,7 @@ - +