Added periodic idmapper cache size check along with conditional flush mechanism. Ran a lot of tests and stress tests to get statistics on usage.

This commit is contained in:
Griatch 2014-05-18 18:28:10 +02:00
parent a143733ccf
commit f2c75bd0f6
8 changed files with 145 additions and 33 deletions

View file

@ -49,12 +49,12 @@ if __name__ == "__main__":
vmem = data[:,2]
nobj = data[:,3]
# count total amount of objects
#ntot = data[:,3].copy()
# calculate derivative of obj creation
oderiv = (0.5*(nobj[2:] - nobj[:-2]) / (secs[2:] - secs[:-2])).copy()
fig = pp.figure()
ax1 = fig.add_subplot(111)
ax1.set_title("Memory usage")
ax1.set_title("Memory usage (200 bots, auto-flush at RMEM ~ 200MB)")
ax1.set_xlabel("Time (mins)")
ax1.set_ylabel("Memory usage (MB)")
ax1.plot(secs, rmem, "r", label="RMEM", lw=2)
@ -63,9 +63,36 @@ if __name__ == "__main__":
ax2 = ax1.twinx()
ax2.plot(secs, nobj, "g--", label="objs in cache", lw=2)
#ax2.plot(secs, ntot, "r--", label="objs total", lw=2)
#ax2.plot(secs[:-2], oderiv/60.0, "g--", label="Objs/second", lw=2)
ax2.set_ylabel("Number of objects")
ax2.legend(loc="lower right")
ax2.annotate("idmapper\nflush", xy=(70,480))
ax2.annotate("@reload", xy=(185,600))
#ax2.annotate("All bots\nfinished\nconnecting", xy=(10, 16900))
#ax2.annotate("idmapper\nflush", xy=(70,480))
#ax2.annotate("@reload", xy=(185,600))
# # plot mem vs cachesize
# nobj, rmem, vmem = nobj[:262].copy(), rmem[:262].copy(), vmem[:262].copy()
#
# fig = pp.figure()
# ax1 = fig.add_subplot(111)
# ax1.set_title("Memory usage per cache size")
# ax1.set_xlabel("Cache size (number of objects)")
# ax1.set_ylabel("Memory usage (MB)")
# ax1.plot(nobj, rmem, "r", label="RMEM", lw=2)
# ax1.plot(nobj, vmem, "b", label="VMEM", lw=2)
#
## # empirical estimate of memory usage: rmem = 35.0 + 0.0157 * Ncache
## # Ncache = int((rmem - 35.0) / 0.0157) (rmem in MB)
#
# rderiv_aver = 0.0157
# fig = pp.figure()
# ax1 = fig.add_subplot(111)
# ax1.set_title("Relation between memory and cache size")
# ax1.set_xlabel("Memory usage (MB)")
# ax1.set_ylabel("Idmapper Cache Size (number of objects)")
# rmem = numpy.linspace(35, 2000, 2000)
# nobjs = numpy.array([int((mem - 35.0) / 0.0157) for mem in rmem])
# ax1.plot(rmem, nobjs, "r", lw=2)
pp.show()