Redis作为一个高性能的内存NoSQL数据库,其容量受到最大内存限制的限制。
用户在使用阿里云Redis时,除了对性能,稳定性有很高的要求外,对内存占用也比较敏感。在使用过程中,有些用户会觉得自己的线上实例内存占用比自己预想的要大。
事实上,实例中的内存除了保存原始的键值对所需的开销外,还有一些运行时产生的额外内存,包括:
本系列文章主要分析这些在Redis中产生的原因,带来的影响和规避的方式。
本文主要分析第一项Redis过期策略对内存的影响。
为了防止一次性清理大量过期Key导致Redis服务受影响,Redis只在空闲时清理过期Key。
具体Redis逐出过期Key的时机为:
- robj lookupKeyRead(redisDb db, robj *key) {
- robj *val;
- expireIfNeeded(db,key);
- val = lookupKey(db,key);
- ...
- return val;
- }
- aeCreateTimeEvent(server.el, 1, serverCron, NULL, NULL)
- int serverCron(struct aeEventLoop eventLoop, long long id, void clientData) {
- ...
- databasesCron();
- ...
- }
- void databasesCron(void) {
- /* Expire keys by random sampling. Not required for slaves
- + as master will synthesize DELs for us. */
- if (server.active_expire_enabled && server.masterhost == NULL)
- activeExpireCycle(ACTIVE_EXPIRE_CYCLE_SLOW);
- ...
- }
- void aeMain(aeEventLoop *eventLoop) {
- eventLoop->stop = 0;
- while (!eventLoop->stop) {
- if (eventLoop->beforesleep != NULL)
- eventLoop->beforesleep(eventLoop);
- aeProcessEvents(eventLoop, AE_ALL_EVENTS);
- }
- }
- void beforeSleep(struct aeEventLoop *eventLoop) {
- ...
- /* Run a fast expire cycle (the called function will return
- - ASAP if a fast cycle is not needed). */
- if (server.active_expire_enabled && server.masterhost == NULL)
- activeExpireCycle(ACTIVE_EXPIRE_CYCLE_FAST);
- ...
- }
Redis过期Key清理的机制对清理的频率和最大时间都有限制,在尽量不影响正常服务的情况下,进行过期Key的清理,以达到长时间服务的性能最优.
Redis会周期性的随机测试一批设置了过期时间的key并进行处理。测试到的已过期的key将被删除。具体的算法如下:
这是一个基于概率的简单算法,基本的假设是抽出的样本能够代表整个key空间,redis持续清理过期的数据直至将要过期的key的百分比降到了25%以下。这也意味着在长期来看任何给定的时刻已经过期但仍占据着内存空间的key的量最多为每秒的写操作量除以4.
代码分析如下:
- void activeExpireCycle(int type) {...
- /* We can use at max ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC percentage of CPU time
- * per iteration. Since this function gets called with a frequency of
- * server.hz times per second, the following is the max amount of
- microseconds we can spend in this function. /
- // 最多允许25%的CPU时间用于过期Key清理
- // 若hz=1,则一次activeExpireCycle最多只能执行250ms
- // 若hz=10,则一次activeExpireCycle最多只能执行25ms
- timelimit = 1000000*ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC/server.hz/100;
- ...
- // 遍历所有db
- for (j = 0; j < dbs_per_call; j++) {
- int expired;
- redisDb *db = server.db+(current_db % server.dbnum);
- /* Increment the DB now so we are sure if we run out of time
- * in the current DB we'll restart from the next. This allows to
- distribute the time evenly across DBs. /
- current_db++;
- /* Continue to expire if at the end of the cycle more than 25%
- of the keys were expired. /
- do {
- ...
- // 一次取20个Key,判断是否过期
- if (num > ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP)
- num = ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP;
- while (num--) {
- dictEntry *de;
- long long ttl;
- if ((de = dictGetRandomKey(db->expires)) == NULL) break;
- ttl = dictGetSignedIntegerVal(de)-now;
- if (activeExpireCycleTryExpire(db,de,now)) expired++;
- }
- if ((iteration & 0xf) == 0) { / check once every 16 iterations. /
- long long elapsed = ustime()-start;
- latencyAddSampleIfNeeded("expire-cycle",elapsed/1000);
- if (elapsed > timelimit) timelimit_exit = 1;
- }
- if (timelimit_exit) return;
- /* We don't repeat the cycle if there are less than 25% of keys
- found expired in the current DB. /
- // 若有5个以上过期Key,则继续直至时间超过25%的CPU时间
- // 若没有5个过期Key,则跳过。
- } while (expired > ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP/4);
- }
- }*/
- // 执行命令
- int processCommand(redisClient * c) {...
- /* Handle the maxmemory directive.
- **
- First we try to free some memory if possible (if there are volatile
- * keys in the dataset). If there are not the only thing we can do
- is returning an error. /
- if (server.maxmemory) {
- int retval = freeMemoryIfNeeded();
- ...
- }
- ...
- }*/
在逐出算法中,根据用户设置的逐出策略,选出待逐出的key,直到当前内存小于最大内存值为主.
可选逐出策略如下:
具体代码如下
- int freeMemoryIfNeeded() {
- ...
- // 计算mem_used
- mem_used = zmalloc_used_memory();
- ...
- / Check if we are over the memory limit. /
- if (mem_used <= server.maxmemory) return REDIS_OK;
- // 如果禁止逐出,返回错误
- if (server.maxmemory_policy == REDIS_MAXMEMORY_NO_EVICTION)
- return REDIS_ERR; / We need to free memory, but policy forbids. /
- mem_freed = 0;
- mem_tofree = mem_used - server.maxmemory;
- long long start = ustime();
- latencyStartMonitor(latency);
- while (mem_freed < mem_tofree) {
- int j, k, keys_freed = 0;
- for (j = 0; j < server.dbnum; j++) {
- // 根据逐出策略的不同,选出待逐出的数据
- long bestval = 0; / just to prevent warning /
- sds bestkey = NULL;
- struct dictEntry *de;
- redisDb *db = server.db+j;
- dict *dict;
- if (server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_LRU ||
- server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_RANDOM)
- {
- dict = server.db[j].dict;
- } else {
- dict = server.db[j].expires;
- }
- if (dictSize(dict) == 0) continue;
- / volatile-random and allkeys-random policy /
- if (server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_RANDOM ||
- server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_RANDOM)
- {
- de = dictGetRandomKey(dict);
- bestkey = dictGetKey(de);
- }
- / volatile-lru and allkeys-lru policy /
- else if (server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_LRU ||
- server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_LRU)
- {
- for (k = 0; k < server.maxmemory_samples; k++) {
- sds thiskey;
- long thisval;
- robj *o;
- de = dictGetRandomKey(dict);
- thiskey = dictGetKey(de);
- /* When policy is volatile-lru we need an additional lookup
- to locate the real key, as dict is set to db->expires. */
- if (server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_LRU)
- de = dictFind(db->dict, thiskey);
- o = dictGetVal(de);
- thisval = estimateObjectIdleTime(o);
- / Higher idle time is better candidate for deletion /
- if (bestkey == NULL || thisval > bestval) {
- bestkey = thiskey;
- bestval = thisval;
- }
- }
- }
- / volatile-ttl /
- else if (server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_TTL) {
- for (k = 0; k < server.maxmemory_samples; k++) {
- sds thiskey;
- long thisval;
- de = dictGetRandomKey(dict);
- thiskey = dictGetKey(de);
- thisval = (long) dictGetVal(de);
- /* Expire sooner (minor expire unix timestamp) is better
- candidate for deletion */
- if (bestkey == NULL || thisval < bestval) {
- bestkey = thiskey;
- bestval = thisval;
- }
- }
- }
- / Finally remove the selected key. */
- // 逐出挑选出的数据
- if (bestkey ) {
- ...
- delta = (long long) zmalloc_used_memory();
- dbDelete(db,keyobj);
- delta -= (long long) zmalloc_used_memory();
- mem_freed += delta;
- ...
- }
- }
- ...
- }
- ...
- return REDIS_OK;
- }
来源: https://yq.aliyun.com/articles/257459