using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading; using Ksat.Supplyment.Library.Model.Uploader; using Ksat.Supplyment.Library.Model; namespace Ksat.Supplyment.Library.Uploader { public class Uploader : IDisposable { /// /// 上传操作执行者 /// internal IUploaderOperator mOperator; /// /// 上传数据队列 /// private readonly ConcurrentQueue> mDatas = new ConcurrentQueue>(); /// /// 交换数据队列 /// private readonly ConcurrentQueue> mSwapDatas = new ConcurrentQueue>(); /// /// 上传请求 /// private readonly LinkedList, UploadModel>> mRequests = new LinkedList, UploadModel>>(); private readonly System.Threading.Thread mDatasWorkerThread; /// /// 最大队列长度 /// public int MaxOperatingQueueSize = 5; /// /// 同时上传的数量 /// public int MaxCocurrentTaskCount = 2; private int mIsCanceled = 0; /// /// 任务唯一标识,与数据库中UploaderID对应 /// private string mTaskIdentify; private Timer mSwapClearTimer; public Uploader(string ID, IUploaderOperator op) { if (op == null) throw new System.ArgumentNullException(); this.mTaskIdentify = ID; this.mOperator = op; //sLog = LoggerFactory.ForContext("Uploader:" + this.mTaskIdentify); mDatasWorkerThread = new System.Threading.Thread(uploadWorker) { Name = "upload worker", IsBackground = true }; mDatasWorkerThread.Start(); mSwapClearTimer = new Timer((x) => { var toDBs = new List>(); UploadModel d; while (mSwapDatas.TryDequeue(out d)) { toDBs.Add(d); } if (toDBs.Count != 0) cacheRequest(toDBs); }, null, 4000, 4000); } public void Dispose() { Interlocked.Exchange(ref mIsCanceled, 1); try { mSwapClearTimer.Dispose(); mDatasWorkerThread.Abort(); List> toCache = new List>(); ///取消当前上传中任务, lock (mRequests) { mRequests.ToList().ForEach((r) => { r.Item1.UploadFinish -= Req_OnRequestFinished; r.Item1.Cancel(); r.Item2.ErrorInfo = "上传程序中止,任务cancel"; toCache.Add(r.Item2); }); } mDatas.ToList().ForEach((d) => toCache.Add(d)); mSwapDatas.ToList().ForEach((d) => toCache.Add(d)); cacheRequest(toCache); } catch { } } /// /// 加入到上传任务队列 /// /// /// private void enqueRequest(TReq req, UploadModel requestModel) where TReq : IUploaderRequest { req.UploadFinish += Req_OnRequestFinished; lock (mRequests) { mRequests.AddLast(new Tuple, UploadModel>(req, requestModel)); } } /// /// 从上传任务队列中移除 /// /// /// private UploadModel dequeRequest(TReq req) where TReq : IUploaderRequest { req.UploadFinish -= Req_OnRequestFinished; UploadModel reqModel = null; lock (mRequests) { var found = mRequests.FirstOrDefault((x) => x.Item1 == req as IUploaderRequest); if (found != null) { reqModel = found.Item2; mRequests.Remove(found); } } return reqModel; } /// /// 任务完成回调 /// /// /// private void Req_OnRequestFinished(object sender, bool success, string reason) { var request = sender as IUploaderRequest; var x = (request.RequestData); var reqModel = dequeRequest(request); if (reqModel == null) { //sLog.Formf($"no request found:{request.RequestData}"); return; } reqModel.ErrorInfo = reason; //sLog.Formf($"{reqModel.Request} finished:{success}"); string cancelRetryReason = null; if (success) using (var db = new CodeFirstDbContext()) { db.UploadFinishs.Add(new UploadFinish { CreateAt = reqModel.CreateAt, RetryAt = reqModel.RetryAt, RetryCount = reqModel.RetryCount, RequestData = mOperator.ConvertRequestToCachel(reqModel.Request), UploaderID = mTaskIdentify }); db.SaveChanges(); } else if (mOperator.ShouldRetryRequest(reqModel, out cancelRetryReason)) { if (mDatas.Count >= MaxOperatingQueueSize) { mSwapDatas.Enqueue(reqModel); } else { mDatas.Enqueue(reqModel); } } else { using (var db = new CodeFirstDbContext()) { db.UploadCancels.Add(new UploadCancel { CreateAt = reqModel.CreateAt, RetryAt = reqModel.RetryAt, RetryCount = reqModel.RetryCount, RequestData = mOperator.ConvertRequestToCachel(reqModel.Request), ErrorInfo = "Task cancel Retry:" + cancelRetryReason ?? "No Reason", UploaderID = mTaskIdentify });; db.SaveChanges(); } } Console.WriteLine($"{DateTime.UtcNow.ToString("mm:ss.fff")} task {x} result:{success}"); //throw new NotImplementedException(); } /// /// 上传worker /// private void uploadWorker() { while (!Interlocked.Equals(mIsCanceled, 1)) { int reqCount; lock (mRequests) reqCount = mRequests.Count; if (reqCount >= MaxCocurrentTaskCount) { System.Threading.Thread.Sleep(10); Console.WriteLine($"{DateTime.UtcNow.ToString("mm:ss.fff")} task exceed {MaxCocurrentTaskCount}, waiting"); continue; } int newRequestCount = MaxCocurrentTaskCount - reqCount; //只尝试加载一次cache bool hasGotCache = false; for (int i = 0; i < newRequestCount;) { UploadModel d; if (mDatas.TryDequeue(out d)) { string donotStartReason; if (!mOperator.ShouldStartRequest(d, out donotStartReason)) { using (var db = new CodeFirstDbContext()) { db.UploadCancels.Add(new UploadCancel { CreateAt = d.CreateAt, RetryAt = d.RetryAt, RetryCount = d.RetryCount, RequestData = mOperator.ConvertRequestToCachel(d.Request), ErrorInfo = "Task cancel Start:" + donotStartReason ?? "No Reason", UploaderID = mTaskIdentify }); ; db.SaveChanges(); } continue; } d.RetryAt = DateTime.Now; d.RetryCount++; var req = mOperator.UploadData(d.Request); enqueRequest(req, d); ThreadPool.QueueUserWorkItem(new WaitCallback((o) => req.Start())); //new System.Threading.Thread(req.Start).Start(); //req.Start(); i++; continue; } else if (!hasGotCache) { hasGotCache = true; var caches = getCaches(MaxOperatingQueueSize); if (caches.Count == 0) { //cache也没有任务了 break; } caches.ForEach((t) => { mDatas.Enqueue(t); }); continue; } } } } private void cacheRequest(List datas) { using (var db = new CodeFirstDbContext()) { //sLog.Formf($"cache requests,{datas.Count()}"); var caches = datas.Select((data) => new UploadCache { UploaderID = mTaskIdentify, CreateAt = DateTime.Now, RetryAt = new DateTime(1970, 1, 1), RetryCount = 0, ErrorInfo = "", RequestData = mOperator.ConvertRequestToCachel(data) } ); db.UploadCaches.AddRange(caches); db.SaveChanges(); } } /// /// 缓存请求数据到数据库 /// /// private void cacheRequest(List> datas) { //sLog.Formf($"cache requests count,{datas.Count()}"); using (var db = new CodeFirstDbContext()) { var caches = datas.Select((data) => new UploadCache { UploaderID = mTaskIdentify, CreateAt = data.CreateAt, RetryAt = data.RetryAt, RetryCount = data.RetryCount, ErrorInfo = data.ErrorInfo, RequestData = mOperator.ConvertRequestToCachel(data.Request) } ); db.UploadCaches.AddRange(caches); db.SaveChanges(); } } /// /// 从数据库中提取指定数量的请求数据 /// /// /// private List> getCaches(int number) { var ret = new List>(); var toGet = number; UploadModel model; while (toGet > 0 && mSwapDatas.TryDequeue(out model)) { ret.Add(model); toGet--; } using (var db = new CodeFirstDbContext()) { var caches = from c in db.UploadCaches orderby c.RetryAt ascending where c.UploaderID == this.mTaskIdentify select c; var x = caches.Take(toGet).ToList(); //从数据库中移除 db.UploadCaches.RemoveRange(x); db.SaveChanges(); var modelsFromDB = x.Select((data) => new UploadModel { CreateAt = data.CreateAt, RetryAt = data.RetryAt, RetryCount = data.RetryCount, ErrorInfo = data.ErrorInfo, Request = mOperator.ConvertCacheToRequest(data.RequestData) } ).ToList(); ret.AddRange(modelsFromDB); } //if (ret.Count() != 0) //sLog.Formf($"get {number} cache return {ret.Count()}"); return ret; } /// /// 添加上传请求 /// public void AddUploadRequest(T data) { if (!Interlocked.Equals(mIsCanceled, 1)) { var requestData = new UploadModel { CreateAt = DateTime.Now, RetryAt = new DateTime(1970, 1, 1), RetryCount = 0, ErrorInfo = "", Request = data }; if (mDatas.Count >= MaxOperatingQueueSize) { //sLog.Formf($"add request to swap,{data}"); mSwapDatas.Enqueue(requestData); } else { //sLog.Formf($"add request to queue,{data}"); mDatas.Enqueue(requestData); } } else { //sLog.Formf("task canceled"); cacheRequest(new List() { data }); } } } }