zggsong
/
s3_demo
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add save to sqlite

Browse Source
master
ZGGSONG 3 years ago
parent 95312349f5
commit 41197f7bab
18 changed files with 951 additions and 8 deletions
Show Stats Download Patch File Download Diff File

27
KsatSupplymentLibrary/App.config
Unescape View File

@ -0,0 +1,27 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<configSections>
<!-- For more information on Entity Framework configuration, visit http://go.microsoft.com/fwlink/?LinkID=237468 -->
<section name="entityFramework" type="System.Data.Entity.Internal.ConfigFile.EntityFrameworkSection, EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" requirePermission="false" />
</configSections>
<entityFramework>
<providers>
<provider invariantName="System.Data.SqlClient" type="System.Data.Entity.SqlServer.SqlProviderServices, EntityFramework.SqlServer" />
<provider invariantName="System.Data.SQLite" type="System.Data.SQLite.EF6.SQLiteProviderServices, System.Data.SQLite.EF6" />
</providers>
</entityFramework>
<connectionStrings>
<add name="sqlite" connectionString="data source=D:\\sqlite.db;BinaryGUID=False" providerName="System.Data.SQLite.EF6" />
</connectionStrings>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
</startup>
<system.data>
<DbProviderFactories>
<remove invariant="System.Data.SQLite.EF6" />
<add name="SQLite Data Provider (Entity Framework 6)" invariant="System.Data.SQLite.EF6" description=".NET Framework Data Provider for SQLite (Entity Framework 6)" type="System.Data.SQLite.EF6.SQLiteProviderFactory, System.Data.SQLite.EF6" />
<remove invariant="System.Data.SQLite" />
<add name="SQLite Data Provider" invariant="System.Data.SQLite" description=".NET Framework Data Provider for SQLite" type="System.Data.SQLite.SQLiteFactory, System.Data.SQLite" />
</DbProviderFactories>
</system.data>
</configuration>

96
KsatSupplymentLibrary/KsatSupplymentLibrary.csproj
Unescape View File

@ -0,0 +1,96 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\EntityFramework.6.4.4\build\EntityFramework.props" Condition="Exists('..\packages\EntityFramework.6.4.4\build\EntityFramework.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{3C90349C-2148-4D79-9881-7B3474EA8AF7}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Ksat.Supplyment.Library</RootNamespace>
<AssemblyName>KsatSupplymentLibrary</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<Deterministic>true</Deterministic>
<NuGetPackageImportStamp>
</NuGetPackageImportStamp>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089, processorArchitecture=MSIL">
<HintPath>..\packages\EntityFramework.6.4.4\lib\net45\EntityFramework.dll</HintPath>
</Reference>
<Reference Include="EntityFramework.SqlServer, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089, processorArchitecture=MSIL">
<HintPath>..\packages\EntityFramework.6.4.4\lib\net45\EntityFramework.SqlServer.dll</HintPath>
</Reference>
<Reference Include="SQLite.CodeFirst, Version=1.7.0.34, Culture=neutral, PublicKeyToken=eb96ba0a78d831a7, processorArchitecture=MSIL">
<HintPath>..\packages\SQLite.CodeFirst.1.7.0.34\lib\net45\SQLite.CodeFirst.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.ComponentModel.DataAnnotations" />
<Reference Include="System.Core" />
<Reference Include="System.Data.Entity" />
<Reference Include="System.Data.SQLite, Version=1.0.115.5, Culture=neutral, PublicKeyToken=db937bc2d44ff139, processorArchitecture=MSIL">
<HintPath>..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.115.5\lib\net45\System.Data.SQLite.dll</HintPath>
</Reference>
<Reference Include="System.Data.SQLite.EF6, Version=1.0.115.5, Culture=neutral, PublicKeyToken=db937bc2d44ff139, processorArchitecture=MSIL">
<HintPath>..\packages\System.Data.SQLite.EF6.1.0.115.5\lib\net45\System.Data.SQLite.EF6.dll</HintPath>
</Reference>
<Reference Include="System.Data.SQLite.Linq, Version=1.0.115.5, Culture=neutral, PublicKeyToken=db937bc2d44ff139, processorArchitecture=MSIL">
<HintPath>..\packages\System.Data.SQLite.Linq.1.0.115.5\lib\net45\System.Data.SQLite.Linq.dll</HintPath>
</Reference>
<Reference Include="System.Runtime.Serialization" />
<Reference Include="System.Security" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Model\Uploader\UploadModel.cs" />
<Compile Include="Model\Uploader\UploadCache.cs" />
<Compile Include="Model\Uploader\UploadCancel.cs" />
<Compile Include="Model\Uploader\UploadFinish.cs" />
<Compile Include="Uploader\IUploaderOperator.cs" />
<Compile Include="Uploader\IUploaderRequest.cs" />
<Compile Include="Uploader\Uploader.cs" />
<None Include="App.config" />
<Compile Include="Model\CodeFirstDbContext.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<None Include="packages.config" />
</ItemGroup>
<ItemGroup>
<Service Include="{508349B6-6B84-4DF5-91F0-309BEEBAD82D}" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>这台计算机上缺少此项目引用的 NuGet 程序包。使用“NuGet 程序包还原”可下载这些程序包。有关更多信息，请参见 http://go.microsoft.com/fwlink/?LinkID=322105。缺少的文件是 {0}。</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\EntityFramework.6.4.4\build\EntityFramework.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\EntityFramework.6.4.4\build\EntityFramework.props'))" />
<Error Condition="!Exists('..\packages\EntityFramework.6.4.4\build\EntityFramework.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\EntityFramework.6.4.4\build\EntityFramework.targets'))" />
<Error Condition="!Exists('..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.115.5\build\net45\Stub.System.Data.SQLite.Core.NetFramework.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.115.5\build\net45\Stub.System.Data.SQLite.Core.NetFramework.targets'))" />
</Target>
<Import Project="..\packages\EntityFramework.6.4.4\build\EntityFramework.targets" Condition="Exists('..\packages\EntityFramework.6.4.4\build\EntityFramework.targets')" />
<Import Project="..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.115.5\build\net45\Stub.System.Data.SQLite.Core.NetFramework.targets" Condition="Exists('..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.115.5\build\net45\Stub.System.Data.SQLite.Core.NetFramework.targets')" />
</Project>

39
KsatSupplymentLibrary/Model/CodeFirstDbContext.cs
Unescape View File

@ -0,0 +1,39 @@
using System;
using System.Data.Entity;
using System.Linq;
namespace Ksat.Supplyment.Library.Model
{
public class CodeFirstDbContext : DbContext
{
//您的上下文已配置为从您的应用程序的配置文件(App.config 或 Web.config)
//使用“CodeFirstDbContext”连接字符串。默认情况下，此连接字符串针对您的 LocalDb 实例上的
//“Ksat.Supplyment.Library.Model.CodeFirstDbContext”数据库。
//
//如果您想要针对其他数据库和/或数据库提供程序，请在应用程序配置文件中修改“CodeFirstDbContext”
//连接字符串。
public CodeFirstDbContext() : base(@"name = sqlite")
{
}
protected override void OnModelCreating(DbModelBuilder modelBuilder)
{
modelBuilder.Entity<Model.Uploader.UploadCache>().ToTable("UploadCaches").HasKey(p => p.Id);
modelBuilder.Entity<Model.Uploader.UploadCancel>().ToTable("UploadCancels").HasKey(p => p.Id);
modelBuilder.Entity<Model.Uploader.UploadFinish>().ToTable("UploadFinishs").HasKey(p => p.Id);
Database.SetInitializer(new SQLite.CodeFirst.SqliteCreateDatabaseIfNotExists<CodeFirstDbContext>(modelBuilder));
}
//为您要在模型中包含的每种实体类型都添加 DbSet。有关配置和使用 Code First 模型
//的详细信息，请参阅 https://blog.csdn.net/wucdsg/article/details/78895366
public virtual DbSet<Model.Uploader.UploadCache> UploadCaches { get; set; }
public virtual DbSet<Model.Uploader.UploadCancel> UploadCancels { get; set; }
public virtual DbSet<Model.Uploader.UploadFinish> UploadFinishs { get; set; }
}
//public class MyEntity
//{
// public int Id { get; set; }
// public string Name { get; set; }
//}
}

44
KsatSupplymentLibrary/Model/Uploader/UploadCache.cs
Unescape View File

@ -0,0 +1,44 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Ksat.Supplyment.Library.Model.Uploader
{
public class UploadCache
{
[Key]
public int Id { get; set; }
/// <summary>
/// 属于哪个上传任务
/// </summary>
[Required]
public string UploaderID { get; set; }
[Required]
public DateTime CreateAt { get; set; }
/// <summary>
/// 最后一次上传时间
/// </summary>
public DateTime RetryAt { get; set; }
/// <summary>
/// 尝试次数
/// </summary>
public int RetryCount { get; set; }
/// <summary>
/// 错误信息
/// </summary>
public string ErrorInfo { get; set; }
/// <summary>
/// 上传相关数据
/// </summary>
public string RequestData { get; set; }
}
}

44
KsatSupplymentLibrary/Model/Uploader/UploadCancel.cs
Unescape View File

@ -0,0 +1,44 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Ksat.Supplyment.Library.Model.Uploader
{
public class UploadCancel
{
[Key]
public int Id { get; set; }
/// <summary>
/// 属于哪个上传任务
/// </summary>
[Required]
public string UploaderID { get; set; }
[Required]
public DateTime CreateAt { get; set; }
/// <summary>
/// 最后一次上传时间
/// </summary>
public DateTime RetryAt { get; set; }
/// <summary>
/// 错误信息
/// </summary>
public string ErrorInfo { get; set; }
/// <summary>
/// 尝试次数
/// </summary>
public int RetryCount { get; set; }
/// <summary>
/// 上传相关数据
/// </summary>
public string RequestData { get; set; }
}
}

39
KsatSupplymentLibrary/Model/Uploader/UploadFinish.cs
Unescape View File

@ -0,0 +1,39 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Ksat.Supplyment.Library.Model.Uploader
{
public class UploadFinish
{
[Key]
public int Id { get; set; }
/// <summary>
/// 属于哪个上传任务
/// </summary>
[Required]
public string UploaderID { get; set; }
[Required]
public DateTime CreateAt { get; set; }
/// <summary>
/// 最后一次上传时间
/// </summary>
public DateTime RetryAt { get; set; }
/// <summary>
/// 尝试次数
/// </summary>
public int RetryCount { get; set; }
/// <summary>
/// 上传相关数据
/// </summary>
public string RequestData { get; set; }
}
}

30
KsatSupplymentLibrary/Model/Uploader/UploadModel.cs
Unescape View File

@ -0,0 +1,30 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Ksat.Supplyment.Library.Model.Uploader
{
public class UploadModel<T>
{
public DateTime CreateAt;
/// <summary>
/// 最后一次上传时间
/// </summary>
public DateTime RetryAt;
/// <summary>
/// 尝试次数
/// </summary>
public int RetryCount;
/// <summary>
/// 错误信息
/// </summary>
public string ErrorInfo;
public T Request;
}
}

36
KsatSupplymentLibrary/Properties/AssemblyInfo.cs
Unescape View File

@ -0,0 +1,36 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// 有关程序集的一般信息由以下
// 控制。更改这些特性值可修改
// 与程序集关联的信息。
[assembly: AssemblyTitle("KsatSupplymentLibrary")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("KsatSupplymentLibrary")]
[assembly: AssemblyCopyright("Copyright © 2021")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// 将 ComVisible 设置为 false 会使此程序集中的类型
//对 COM 组件不可见。如果需要从 COM 访问此程序集中的类型
//请将此类型的 ComVisible 特性设置为 true。
[assembly: ComVisible(false)]
// 如果此项目向 COM 公开，则下列 GUID 用于类型库的 ID
[assembly: Guid("3c90349c-2148-4d79-9881-7b3474ea8af7")]
// 程序集的版本信息由下列四个值组成:
//
// 主版本
// 次版本
// 生成号
// 修订号
//
//可以指定所有这些值，也可以使用“生成号”和“修订号”的默认值
//通过使用 "*"，如下所示:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

39
KsatSupplymentLibrary/Uploader/IUploaderOperator.cs
Unescape View File

@ -0,0 +1,39 @@
using Ksat.Supplyment.Library.Model.Uploader;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace Ksat.Supplyment.Library.Uploader
{
public interface IUploaderOperator<T>
{
/// <summary>
/// 上传数据
/// </summary>
IUploaderRequest<T> UploadData(T data);
T ConvertCacheToRequest(string cache);
string ConvertRequestToCachel(T requestData);
/// <summary>
/// 任务开始前，判断任务是否要开始
/// </summary>
/// <param name="requestModel"></param>
/// <param name="cancelReason"></param>
/// <returns></returns>
bool ShouldStartRequest(UploadModel<T> requestModel, out string cancelReason);
/// <summary>
/// 任务失败以后，判断是否需要重试
/// </summary>
/// <param name="requestModel"></param>
/// <param name="cancelReason"></param>
/// <returns></returns>
bool ShouldRetryRequest(UploadModel<T> requestModel, out string cancelReason);
}
}

33
KsatSupplymentLibrary/Uploader/IUploaderRequest.cs
Unescape View File

@ -0,0 +1,33 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Ksat.Supplyment.Library.Model.Uploader;
namespace Ksat.Supplyment.Library.Uploader
{
public delegate void UploadFinishHandler(object sender, bool success, string reason) ;
public interface IUploaderRequest<T>
{
/// <summary>
/// 请求数据
/// </summary>
T RequestData { get; set; }
/// <summary>
/// 开始上传任务
/// </summary>
void Start();
/// <summary>
/// 停止上传任务
/// </summary>
void Cancel();
/// <summary>
/// 上传完成事件
/// </summary>
event UploadFinishHandler UploadFinish;
}
}

412
KsatSupplymentLibrary/Uploader/Uploader.cs
Unescape View File

@ -0,0 +1,412 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using Ksat.Supplyment.Library.Model.Uploader;
using Ksat.Supplyment.Library.Model;
namespace Ksat.Supplyment.Library.Uploader
{
public class Uploader<T> : IDisposable
{
/// <summary>
/// 上传操作执行者
/// </summary>
internal IUploaderOperator<T> mOperator;
/// <summary>
/// 上传数据队列
/// </summary>
private readonly ConcurrentQueue<UploadModel<T>> mDatas = new ConcurrentQueue<UploadModel<T>>();
/// <summary>
/// 交换数据队列
/// </summary>
private readonly ConcurrentQueue<UploadModel<T>> mSwapDatas = new ConcurrentQueue<UploadModel<T>>();
/// <summary>
/// 上传请求
/// </summary>
private readonly LinkedList<Tuple<IUploaderRequest<T>, UploadModel<T>>> mRequests = new LinkedList<Tuple<IUploaderRequest<T>, UploadModel<T>>>();
private readonly System.Threading.Thread mDatasWorkerThread;
/// <summary>
/// 最大队列长度
/// </summary>
public int MaxOperatingQueueSize = 5;
/// <summary>
/// 同时上传的数量
/// </summary>
public int MaxCocurrentTaskCount = 2;
private int mIsCanceled = 0;
/// <summary>
/// 任务唯一标识，与数据库中UploaderID对应
/// </summary>
private string mTaskIdentify;
private Timer mSwapClearTimer;
public Uploader(string ID, IUploaderOperator<T> op)
{
if (op == null)
throw new System.ArgumentNullException();
this.mTaskIdentify = ID;
this.mOperator = op;
//sLog = LoggerFactory.ForContext("Uploader:" + this.mTaskIdentify);
mDatasWorkerThread = new System.Threading.Thread(uploadWorker)
{
Name = "upload worker",
IsBackground = true
};
mDatasWorkerThread.Start();
mSwapClearTimer = new Timer((x) =>
{
var toDBs = new List<UploadModel<T>>();
UploadModel<T> d;
while (mSwapDatas.TryDequeue(out d))
{
toDBs.Add(d);
}
if (toDBs.Count != 0)
cacheRequest(toDBs);
}, null, 4000, 4000);
}
public void Dispose()
{
Interlocked.Exchange(ref mIsCanceled, 1);
try
{
mSwapClearTimer.Dispose();
mDatasWorkerThread.Abort();
List<UploadModel<T>> toCache = new List<UploadModel<T>>();
///取消当前上传中任务，
lock (mRequests)
{
mRequests.ToList().ForEach((r) =>
{
r.Item1.UploadFinish -= Req_OnRequestFinished;
r.Item1.Cancel();
r.Item2.ErrorInfo = "上传程序中止，任务cancel";
toCache.Add(r.Item2);
});
}
mDatas.ToList().ForEach((d) => toCache.Add(d));
mSwapDatas.ToList().ForEach((d) => toCache.Add(d));
cacheRequest(toCache);
}
catch
{
}
}
/// <summary>
/// 加入到上传任务队列
/// </summary>
/// <typeparam name="TReq"></typeparam>
/// <param name="req"></param>
private void enqueRequest<TReq>(TReq req, UploadModel<T> requestModel) where TReq : IUploaderRequest<T>
{
req.UploadFinish += Req_OnRequestFinished;
lock (mRequests)
{
mRequests.AddLast(new Tuple<IUploaderRequest<T>, UploadModel<T>>(req, requestModel));
}
}
/// <summary>
/// 从上传任务队列中移除
/// </summary>
/// <typeparam name="TReq"></typeparam>
/// <param name="req"></param>
private UploadModel<T> dequeRequest<TReq>(TReq req) where TReq : IUploaderRequest<T>
{
req.UploadFinish -= Req_OnRequestFinished;
UploadModel<T> reqModel = null;
lock (mRequests)
{
var found = mRequests.FirstOrDefault((x) => x.Item1 == req as IUploaderRequest<T>);
if (found != null)
{
reqModel = found.Item2;
mRequests.Remove(found);
}
}
return reqModel;
}
/// <summary>
/// 任务完成回调
/// </summary>
/// <param name="sender"></param>
/// <param name="success"></param>
private void Req_OnRequestFinished(object sender, bool success, string reason)
{
var request = sender as IUploaderRequest<T>;
var x = (request.RequestData);
var reqModel = dequeRequest(request);
if (reqModel == null)
{
//sLog.Formf($"no request found:{request.RequestData}");
return;
}
reqModel.ErrorInfo = reason;
//sLog.Formf($"{reqModel.Request} finished:{success}");
string cancelRetryReason = null;
if (success)
using (var db = new CodeFirstDbContext())
{
db.UploadFinishs.Add(new UploadFinish
{
CreateAt = reqModel.CreateAt,
RetryAt = reqModel.RetryAt,
RetryCount = reqModel.RetryCount,
RequestData = mOperator.ConvertRequestToCachel(reqModel.Request),
UploaderID = mTaskIdentify
});
db.SaveChanges();
}
else if (mOperator.ShouldRetryRequest(reqModel, out cancelRetryReason))
{
if (mDatas.Count >= MaxOperatingQueueSize)
{
mSwapDatas.Enqueue(reqModel);
}
else
{
mDatas.Enqueue(reqModel);
}
}
else
{
using (var db = new CodeFirstDbContext())
{
db.UploadCancels.Add(new UploadCancel
{
CreateAt = reqModel.CreateAt,
RetryAt = reqModel.RetryAt,
RetryCount = reqModel.RetryCount,
RequestData = mOperator.ConvertRequestToCachel(reqModel.Request),
ErrorInfo = "Task cancel Retry:" + cancelRetryReason ?? "No Reason",
UploaderID = mTaskIdentify
});;
db.SaveChanges();
}
}
Console.WriteLine($"{DateTime.UtcNow.ToString("mm:ss.fff")} task {x} result:{success}");
//throw new NotImplementedException();
}
/// <summary>
/// 上传worker
/// </summary>
private void uploadWorker()
{
while (!Interlocked.Equals(mIsCanceled, 1))
{
int reqCount;
lock (mRequests)
reqCount = mRequests.Count;
if (reqCount >= MaxCocurrentTaskCount)
{
System.Threading.Thread.Sleep(10);
Console.WriteLine($"{DateTime.UtcNow.ToString("mm:ss.fff")} task exceed {MaxCocurrentTaskCount}, waiting");
continue;
}
int newRequestCount = MaxCocurrentTaskCount - reqCount;
//只尝试加载一次cache
bool hasGotCache = false;
for (int i = 0; i < newRequestCount;)
{
UploadModel<T> d;
if (mDatas.TryDequeue(out d))
{
string donotStartReason;
if (!mOperator.ShouldStartRequest(d, out donotStartReason))
{
using (var db = new CodeFirstDbContext())
{
db.UploadCancels.Add(new UploadCancel
{
CreateAt = d.CreateAt,
RetryAt = d.RetryAt,
RetryCount = d.RetryCount,
RequestData = mOperator.ConvertRequestToCachel(d.Request),
ErrorInfo = "Task cancel Start:" + donotStartReason ?? "No Reason",
UploaderID = mTaskIdentify
}); ;
db.SaveChanges();
}
continue;
}
d.RetryAt = DateTime.Now;
d.RetryCount++;
var req = mOperator.UploadData(d.Request);
enqueRequest(req, d);
ThreadPool.QueueUserWorkItem(new WaitCallback((o) => req.Start()));
//new System.Threading.Thread(req.Start).Start();
//req.Start();
i++;
continue;
}
else if (!hasGotCache)
{
hasGotCache = true;
var caches = getCaches(MaxOperatingQueueSize);
if (caches.Count == 0)
{
//cache也没有任务了
break;
}
caches.ForEach((t) => { mDatas.Enqueue(t); });
continue;
}
}
}
}
private void cacheRequest(List<T> datas)
{
using (var db = new CodeFirstDbContext())
{
//sLog.Formf($"cache requests,{datas.Count()}");
var caches = datas.Select((data) =>
new UploadCache
{
UploaderID = mTaskIdentify,
CreateAt = DateTime.Now,
RetryAt = new DateTime(1970, 1, 1),
RetryCount = 0,
ErrorInfo = "",
RequestData = mOperator.ConvertRequestToCachel(data)
}
);
db.UploadCaches.AddRange(caches);
db.SaveChanges();
}
}
/// <summary>
/// 缓存请求数据到数据库
/// </summary>
/// <param name="datas"></param>
private void cacheRequest(List<UploadModel<T>> datas)
{
//sLog.Formf($"cache requests count,{datas.Count()}");
using (var db = new CodeFirstDbContext())
{
var caches = datas.Select((data) =>
new UploadCache
{
UploaderID = mTaskIdentify,
CreateAt = data.CreateAt,
RetryAt = data.RetryAt,
RetryCount = data.RetryCount,
ErrorInfo = data.ErrorInfo,
RequestData = mOperator.ConvertRequestToCachel(data.Request)
}
);
db.UploadCaches.AddRange(caches);
db.SaveChanges();
}
}
/// <summary>
/// 从数据库中提取指定数量的请求数据
/// </summary>
/// <param name="number"></param>
/// <returns></returns>
private List<UploadModel<T>> getCaches(int number)
{
var ret = new List<UploadModel<T>>();
var toGet = number;
UploadModel<T> model;
while (toGet > 0 && mSwapDatas.TryDequeue(out model))
{
ret.Add(model);
toGet--;
}
using (var db = new CodeFirstDbContext())
{
var caches = from c in db.UploadCaches
orderby c.RetryAt ascending
where c.UploaderID == this.mTaskIdentify
select c;
var x = caches.Take(toGet).ToList();
//从数据库中移除
db.UploadCaches.RemoveRange(x);
db.SaveChanges();
var modelsFromDB = x.Select((data) =>
new UploadModel<T>
{
CreateAt = data.CreateAt,
RetryAt = data.RetryAt,
RetryCount = data.RetryCount,
ErrorInfo = data.ErrorInfo,
Request = mOperator.ConvertCacheToRequest(data.RequestData)
}
).ToList();
ret.AddRange(modelsFromDB);
}
//if (ret.Count() != 0)
//sLog.Formf($"get {number} cache return {ret.Count()}");
return ret;
}
/// <summary>
/// 添加上传请求
/// </summary>
public void AddUploadRequest(T data)
{
if (!Interlocked.Equals(mIsCanceled, 1))
{
var requestData = new UploadModel<T>
{
CreateAt = DateTime.Now,
RetryAt = new DateTime(1970, 1, 1),
RetryCount = 0,
ErrorInfo = "",
Request = data
};
if (mDatas.Count >= MaxOperatingQueueSize)
{
//sLog.Formf($"add request to swap,{data}");
mSwapDatas.Enqueue(requestData);
}
else
{
//sLog.Formf($"add request to queue,{data}");
mDatas.Enqueue(requestData);
}
}
else
{
//sLog.Formf("task canceled");
cacheRequest(new List<T>() { data });
}
}
}
}

1
KsatSupplymentLibrary/flow.drawio
Unescape View File

@ -0,0 +1 @@
<mxfile host="Electron" modified="2021-12-02T08:51:57.547Z" agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/14.9.6 Chrome/89.0.4389.128 Electron/12.0.16 Safari/537.36" etag="woK_veF71DVR6v2Ub5GA" version="14.9.6" type="device"><diagram id="gKco9Rzae7YVhnx6VEaf" name="第 1 页">5VzbcqM4EP0aHrOFkATiEdt4tmr2MjV52NmnLWIrNrtgORjH9nz9SkKYm5wh4wTZTtWUB1oIwVH3Uau7iQXH6f5TFq2Xv7M5TSzHnu8tOLEcB2AM+H9CclASUEoWWTxXskpwH3+nSmgr6Tae003jwpyxJI/XTeGMrVZ0ljdkUZaxXfOyR5Y0R11HC9oR3M+ipCv9K57ny0JKHK+S/0rjxbIcGbh+0ZJG5cXqTTbLaM52NREMLTjOGMuLo3Q/polAr8Sl6Dc90Xp8sIyu8j4d9p8/fU3/+YxX0bf7pzD6cvc13NzB4i7PUbJVL6weNj+UCNA5B0SdsixfsgVbRUlYSUcZ267mVAxj87Pqmt8YW3Mh4MJ/aZ4f1OxG25xx0TJPE9VajCkGOvluSrRh22xGX3ihUkeibEHzF65zjjPAdZeylObZgffLaBLl8XPzOSKlQ4vjdRXM/EAh/QrUQRf10LVIaAVYHPjIGrlWyH9H4l+ILRJYAejMTIW7AHG3jHN6v44kPDtujjqMn2mW0/3LKHdRUR0colS5NGakzneVZYBS3Zc1q3DtdwISapC8bv11euovMqm/jlZ/R15NW22pv0Qci4NxKblsjT6eG9No99YUGvVUaGxSoaFGoz8G7J5J2JGWR3xiBVPBEb5tBa44EMzCmyS/ENnEf31Xkg5fN4kVTgXF+NMuoSxZ+rDdDEIm0G6SCdSQCdFwCXkvLsEaeDkFc+gmAsNgIqAWYBJrZKeTKI+MEzJCLQx904QMb85F9npSg2+SGjy97rYcCl9osD9VSuzzTm6UCq1c8BcbgdIgL4QQkEaZByUEouXbtt+GBf2Oxpsdh+TS+ADbpvnA12BIJKmOdPp4YQ6u65nGD2oW/evm0zJU9UNCBSfmaqCgg90BfrtOWDTnsqct5TLTyur6FxdfuLntGOgbIINmN2Sa9f+DAA+gUZbQxSavZQfmgVY4xzXscIHb4w/YV42NRiihxtX9IMAXb2qMPzTbZcEfE8uHkjZ8i0w0kWAR9+GX2c0dHL8BuihGMR7TAfja9bhvZPJsb1l2DbIsOtQuWLN4lW9qd/4iBNWE++0Jt+3WnBV3rGbw+GhnTKouEOqJAB03F2E9oUUwsNNN1xYGdtGJ14LHeHwOdD3FVP5no7uVcbzaWxpkPMGEutvAa2cU0pNRoNGIJtBF3+Q6SMIi+F4EOINJzb2+uIwpbgeUjBMA6noI16W/Tt/40dme3U+tiKhF+S4YYEV0dCzlJsInfGTy6WYsYZlscZ+2TDqLDvRo5FK7LnILP7JIGah0lwzOlqYnw9xHg1Nj8Gcuhim7y90tGZftD1njzuI+lQcrSnkmVhDKlTuQfm/NsGVsXQ7oiVaNJ1yMRdRDBUEp8Y0bP2olNHzj0XjYzfBcmfH3LlgzYvzAdvVTPu3bYRi6OBG14lZTJLQ8i4x0q6md3t0ZNyvSMiti3KwcI7tMDld2+Cb6/wIJLAV/S4HN/XwlmOzVEMXZoX72hWYxh4BmSviWlur0tNTC0TPlZpaP2dgPNYIqHJfLCtNiTVJy0KAK1FXK+II7CvpQS7XciIv12JO1ul0Q+VDxekOHccRhE0THeGYXdTXvyhdn6Pc0eWQ0s4tuLsvQG/hTRjJQlkFXDOKKug9ttbNy6GvpM5Ms7La8Ds90dRK6uUo71Nu5t/VTNRB/XHtOoTfOZyclf2oT5baK2Hzy8h7Khy9e/z5bKKTbQrki66+yczIC8dpohwpjygI5oimHM8h3xDTf4ZuLxKO+pcXI6BYJnSgubq3WsthF6LMrCuZ1pS0DxwkAaMVWzBfIYQ1rXLkO980mIaPZJKSv5RZVx7bSYVGHfAlVWQC0v7Izvd9HOr+9myE4XdZ9zAfUv1ISTX6t1+kV8iwieYyTZHxMfsBHMqOzGZdv8oz9R2stDwQjbL/RHLa/ZTD/pSS+uRIv3LfECxutrcPdfVqZRAuQRZBYOblRkO72dsj4GHBaiyXSFINqNfb9HL6bK2rGvb/uNVpNjnX1VeWeRBB/wLVV6m3AVVfGfvnvuR6fGaJ2Wul6bLxAA2uI4Lq0HA/FyydyqqhVRVbmVI/3KJ5Mdaum69XZ3nath4d/kO3tFIc4A4Qq8Ks+bH43R+xNrLXlGnuOaWt1NVEJzUf5RPwSUH2LazqneGnRbLe7M/7jzw5E/IXzJhZREi9W/HjGX1yksUcClngWJYFqSOP5vKBIuom/Rw/yVoIklXXz++KRhSfiXpwVNwVBgs7Cs2Ir+jbYwzJ/WpKApn4BarB3Xo89P63+YlbBJ9UfHoPh/w==</diagram></mxfile>

10
KsatSupplymentLibrary/packages.config
Unescape View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="EntityFramework" version="6.4.4" targetFramework="net45" />
<package id="SQLite.CodeFirst" version="1.7.0.34" targetFramework="net45" />
<package id="Stub.System.Data.SQLite.Core.NetFramework" version="1.0.115.5" targetFramework="net45" />
<package id="System.Data.SQLite" version="1.0.115.5" targetFramework="net45" />
<package id="System.Data.SQLite.Core" version="1.0.115.5" targetFramework="net45" />
<package id="System.Data.SQLite.EF6" version="1.0.115.5" targetFramework="net45" />
<package id="System.Data.SQLite.Linq" version="1.0.115.5" targetFramework="net45" />
</packages>

6
S3Demo.sln
Unescape View File

@ -5,6 +5,8 @@ VisualStudioVersion = 16.0.31702.278
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "S3Demo", "S3Demo\S3Demo.csproj", "{10785A7F-B71E-4576-BD28-3979CD9A005C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KsatSupplymentLibrary", "KsatSupplymentLibrary\KsatSupplymentLibrary.csproj", "{3C90349C-2148-4D79-9881-7B3474EA8AF7}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -15,6 +17,10 @@ Global
{10785A7F-B71E-4576-BD28-3979CD9A005C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{10785A7F-B71E-4576-BD28-3979CD9A005C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{10785A7F-B71E-4576-BD28-3979CD9A005C}.Release|Any CPU.Build.0 = Release|Any CPU
{3C90349C-2148-4D79-9881-7B3474EA8AF7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3C90349C-2148-4D79-9881-7B3474EA8AF7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3C90349C-2148-4D79-9881-7B3474EA8AF7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3C90349C-2148-4D79-9881-7B3474EA8AF7}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

21
S3Demo/App.config
Unescape View File

@ -1,6 +1,25 @@
<?xml version="1.0" encoding="utf-8" ?>
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<configSections>
<!-- For more information on Entity Framework configuration, visit http://go.microsoft.com/fwlink/?LinkID=237468 -->
<section name="entityFramework" type="System.Data.Entity.Internal.ConfigFile.EntityFrameworkSection, EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" requirePermission="false" />
</configSections>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
</startup>
<entityFramework>
<providers>
<provider invariantName="System.Data.SqlClient" type="System.Data.Entity.SqlServer.SqlProviderServices, EntityFramework.SqlServer" />
<provider invariantName="System.Data.SQLite" type="System.Data.SQLite.EF6.SQLiteProviderServices, System.Data.SQLite.EF6" />
</providers>
</entityFramework>
<connectionStrings>
<add name="sqlite" connectionString="data source=D:\\sqlite.db;BinaryGUID=False" providerName="System.Data.SQLite.EF6" />
</connectionStrings>
<system.data>
<DbProviderFactories>
<remove invariant="System.Data.SQLite.EF6" />
<add name="SQLite Data Provider (Entity Framework 6)" invariant="System.Data.SQLite.EF6" description=".NET Framework Data Provider for SQLite (Entity Framework 6)" type="System.Data.SQLite.EF6.SQLiteProviderFactory, System.Data.SQLite.EF6" />
<remove invariant="System.Data.SQLite" /><add name="SQLite Data Provider" invariant="System.Data.SQLite" description=".NET Framework Data Provider for SQLite" type="System.Data.SQLite.SQLiteFactory, System.Data.SQLite" /></DbProviderFactories>
</system.data>
</configuration>

20
S3Demo/Run.cs
Unescape View File

@ -8,6 +8,7 @@ using System.Threading.Tasks;
using System.Diagnostics;
using System.Text.RegularExpressions;
using System.IO;
using Ksat.Supplyment.Library.Model;
namespace S3Demo
{
@ -45,8 +46,27 @@ namespace S3Demo
private static string[] paths = { @"D:\ClientDir" };
private static string suffix = "*.png";
private static bool incsubdir = true;
public static void Main()
{
/// 写入数据库操作
//using (var dev = new CodeFirstDbContext())
//{
// dev.UploadCaches.Add(new Ksat.Supplyment.Library.Model.Uploader.UploadCache() {
// Id = 1,
// UploaderID = "02",
// CreateAt = DateTime.Now.AddHours(-1),
// RetryAt = DateTime.Now,
// RetryCount = 1,
// ErrorInfo = "null",
// RequestData = "testInfo"
// });
// var i = dev.SaveChanges();
// Console.WriteLine(i);
//}
try
{
//FileModel.onFileChange(paths, suffix, true);

41
S3Demo/S3Demo.csproj
Unescape View File

@ -1,5 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\EntityFramework.6.4.4\build\EntityFramework.props" Condition="Exists('..\packages\EntityFramework.6.4.4\build\EntityFramework.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -11,6 +12,8 @@
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<Deterministic>true</Deterministic>
<NuGetPackageImportStamp>
</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup>
<LangVersion>preview</LangVersion>
@ -41,8 +44,27 @@
<Reference Include="AWSSDK.S3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=885c28607f98e604, processorArchitecture=MSIL">
<HintPath>..\packages\AWSSDK.S3.3.7.5\lib\net45\AWSSDK.S3.dll</HintPath>
</Reference>
<Reference Include="EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089, processorArchitecture=MSIL">
<HintPath>..\packages\EntityFramework.6.4.4\lib\net45\EntityFramework.dll</HintPath>
</Reference>
<Reference Include="EntityFramework.SqlServer, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089, processorArchitecture=MSIL">
<HintPath>..\packages\EntityFramework.6.4.4\lib\net45\EntityFramework.SqlServer.dll</HintPath>
</Reference>
<Reference Include="SQLite.CodeFirst, Version=1.7.0.34, Culture=neutral, PublicKeyToken=eb96ba0a78d831a7, processorArchitecture=MSIL">
<HintPath>..\packages\SQLite.CodeFirst.1.7.0.34\lib\net45\SQLite.CodeFirst.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.ComponentModel.DataAnnotations" />
<Reference Include="System.Core" />
<Reference Include="System.Data.SQLite, Version=1.0.115.5, Culture=neutral, PublicKeyToken=db937bc2d44ff139, processorArchitecture=MSIL">
<HintPath>..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.115.5\lib\net45\System.Data.SQLite.dll</HintPath>
</Reference>
<Reference Include="System.Data.SQLite.EF6, Version=1.0.115.5, Culture=neutral, PublicKeyToken=db937bc2d44ff139, processorArchitecture=MSIL">
<HintPath>..\packages\System.Data.SQLite.EF6.1.0.115.5\lib\net45\System.Data.SQLite.EF6.dll</HintPath>
</Reference>
<Reference Include="System.Data.SQLite.Linq, Version=1.0.115.5, Culture=neutral, PublicKeyToken=db937bc2d44ff139, processorArchitecture=MSIL">
<HintPath>..\packages\System.Data.SQLite.Linq.1.0.115.5\lib\net45\System.Data.SQLite.Linq.dll</HintPath>
</Reference>
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
@ -67,5 +89,24 @@
<ItemGroup>
<Analyzer Include="..\packages\AWSSDK.S3.3.7.5\analyzers\dotnet\cs\AWSSDK.S3.CodeAnalysis.dll" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\KsatSupplymentLibrary\KsatSupplymentLibrary.csproj">
<Project>{3c90349c-2148-4d79-9881-7b3474ea8af7}</Project>
<Name>KsatSupplymentLibrary</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<WCFMetadata Include="Connected Services\" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>这台计算机上缺少此项目引用的 NuGet 程序包。使用“NuGet 程序包还原”可下载这些程序包。有关更多信息，请参见 http://go.microsoft.com/fwlink/?LinkID=322105。缺少的文件是 {0}。</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\EntityFramework.6.4.4\build\EntityFramework.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\EntityFramework.6.4.4\build\EntityFramework.props'))" />
<Error Condition="!Exists('..\packages\EntityFramework.6.4.4\build\EntityFramework.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\EntityFramework.6.4.4\build\EntityFramework.targets'))" />
<Error Condition="!Exists('..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.115.5\build\net45\Stub.System.Data.SQLite.Core.NetFramework.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.115.5\build\net45\Stub.System.Data.SQLite.Core.NetFramework.targets'))" />
</Target>
<Import Project="..\packages\EntityFramework.6.4.4\build\EntityFramework.targets" Condition="Exists('..\packages\EntityFramework.6.4.4\build\EntityFramework.targets')" />
<Import Project="..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.115.5\build\net45\Stub.System.Data.SQLite.Core.NetFramework.targets" Condition="Exists('..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.115.5\build\net45\Stub.System.Data.SQLite.Core.NetFramework.targets')" />
</Project>

7
S3Demo/packages.config
Unescape View File

@ -2,4 +2,11 @@
<packages>
<package id="AWSSDK.Core" version="3.7.5.2" targetFramework="net45" />
<package id="AWSSDK.S3" version="3.7.5" targetFramework="net45" />
<package id="EntityFramework" version="6.4.4" targetFramework="net45" />
<package id="SQLite.CodeFirst" version="1.7.0.34" targetFramework="net45" />
<package id="Stub.System.Data.SQLite.Core.NetFramework" version="1.0.115.5" targetFramework="net45" />
<package id="System.Data.SQLite" version="1.0.115.5" targetFramework="net45" />
<package id="System.Data.SQLite.Core" version="1.0.115.5" targetFramework="net45" />
<package id="System.Data.SQLite.EF6" version="1.0.115.5" targetFramework="net45" />
<package id="System.Data.SQLite.Linq" version="1.0.115.5" targetFramework="net45" />
</packages>
Write Preview
Loading…
Cancel
Save