wordpress 媒体库管理,自己的网站什么做优化,网盘资源搜索神器,眉县网站开发为什么要用内存池
为什么要用内存池#xff1f;首先#xff0c;在7 * 24h的服务器中如果不使用内存池#xff0c;而使用malloc和free#xff0c;那么就非常容易产生内存碎片#xff0c;早晚都会申请内存失败#xff1b;并且在比较复杂的代码或者继承的屎山中#xff0c…为什么要用内存池
为什么要用内存池首先在7 * 24h的服务器中如果不使用内存池而使用malloc和free那么就非常容易产生内存碎片早晚都会申请内存失败并且在比较复杂的代码或者继承的屎山中非常容易出现内存泄漏导致mmo的问题。
为了解决这两个问题内存池就应运而生了。内存池预先分配一大块内存来做一个内存池业务中的内存分配和释放都由这个内存池来管理内存池内的内存不足时其内部会自己申请。所以内存碎片的问题就交由内存池的算法来优化而内存泄漏的问题只需要遵守内存池提供的api就非常容易避免内存泄漏了。
即使出现了内存泄漏排查的思路也很清晰。1.检查是不是内存池的问题2.如果不是内存池的问题就检查是不是第三方库的内存泄漏。
内存池的使用场景
全局内存池一个连接一个内存池(本文实现这个场景的内存池)
设计一个内存池
总体介绍
由于本文是一个连接一个内存池所以后续介绍和代码都是以4k为分界线大于4k的我们认为是大块内存小于4k的我们认为是小块内存。并且注意这里的4k并不是严格遵照4096而是在描述上用4k比较好描述。
在真正使用内存之前内存池提前分配一定数量且大小相等的内存块以作备用当真正被用户调用api分配内存的时候直接从内存块中获取内存指小块内存当内存块不够用了再有内存池取申请新的内存块。而如果是需要大块内存则内存池直接申请大块内存再返回给用户。
内存池就是将这些提前申请的内存块组织管理起来的数据结构内存池实现原理主要分为分配回收扩容三部分。
内存池原理之小块内存分配 内存池预申请一块4k的内存块这里称为block即block4k内存块。当用户向内存池申请内存size小于4k时内存池从block的空间中划分出去size空间当再有新申请时再划分出去。扩容 直到block中的剩余空间不足以分配size大小那么此时内存池会再次申请一块block再从新的block中划分size空间给用户。回收 每一次申请小内存都会在对应的block中引用计数加1每一次释放小内存时都会在block中引用计数减1只有当引用计数为零的时候才会回收block使他重新成为空闲空间以便重复利用空间。这样内存池避免频繁向内核申请/释放内存从而提高系统性能。
内存池原理之大块内存分配 因为大块内存是大于4k的所以内存池不预先申请内存也就是用户申请的时候内存池再申请内存然后返回给用户。扩容 大块内存不存在扩容。回收 对于大块内存来说回收就直接free掉即可。
上面理论讲完了下面来介绍如何管理小块内存和大块内存。
小块内存的分配与管理
在创建内存池的时候会预先申请一块4k的内存并且在起始处将pool的结构体和node的结构体放进去从last开始一直到end都是空闲内存last , end 中间的区域就用来存储小块内存。每一次mp_malloc就将last指针后移直到 e n d − l a s t s i z e end - last size end−lastsize 时进行扩容将新block的last后移即可。
初始状态 分配内存 扩容 相关视频推荐
200行代码实现slab开启内存池的内存管理准备linux环境
线程池、内存池、异步请求池、数据库连接池、无锁队列的ringbuffer提升程序性能必备技术
5种内存泄漏检测方式让你重新理解C内存管理
免费学习地址C/CLinux服务器开发/后台架构师 需要C/C Linux服务器架构师学习资料加qun812855908获取资料包括C/CLinuxgolang技术NginxZeroMQMySQLRedisfastdfsMongoDBZK流媒体CDNP2PK8SDockerTCP/IP协程DPDKffmpeg等免费分享 大块内存的分配与管理
对于大块内存前面已经说了用户申请的时候内存池才申请
申请一块大内存 再申请一块大内存 内存池代码实现
向外提供的api
mp_create_pool创建一个线程池其核心是创建struct mp_pool_s这个结构体并申请4k内存将各个指针指向上文初始状态的图一样。mp_destroy_pool销毁内存池遍历小块结构体和大块结构体进行free释放内存mp_malloc提供给用户申请内存的apimp_calloc通过mp_malloc申请内存后置零相当于callocmp_free释放由mp_malloc返回的内存mp_reset_pool将block的last置为初始状态销毁所有大块内存monitor_mp_poll监控内存池状态
struct mp_pool_s *mp_create_pool(size_t size);void mp_destroy_pool(struct mp_pool_s *pool);void *mp_malloc(struct mp_pool_s *pool, size_t size);void *mp_calloc(struct mp_pool_s *pool, size_t size);void mp_free(struct mp_pool_s *pool, void *p);void mp_reset_pool(struct mp_pool_s *pool);void monitor_mp_poll(struct mp_pool_s *pool, char *tk);
相关结构体的定义
mp_pool_s 就是整个内存池的管理结构我们做的内存池是一个连接一个内存池所以对于整个程序而言内存池对象是有很多个的。
可能读者会有疑问有了head为什么还有current是因为如果一个block剩余空间小于size超过一定次数后将current指向下一个block这样就加快内存分配效率减少遍历次数。
//每4k一block结点
struct mp_node_s {unsigned char *end;//块的结尾unsigned char *last;//使用到哪了struct mp_node_s *next;//链表int quote;//引用计数int failed;//失效次数
};struct mp_large_s {struct mp_large_s *next;//链表int size;//alloc的大小void *alloc;//大块内存的起始地址
};struct mp_pool_s {struct mp_large_s *large;struct mp_node_s *head;struct mp_node_s *current;
};
内存对齐
访问速度是内存对齐的原因之一另外一个原因是某些平台(arm)不支持未内存对齐的访问
在4k里面划分内存那么必然有很多地方是不对齐的所以这里提供两个内存对齐的函数。那么为什么要内存对齐呢其一提高访问速度其二某些平台arm不支持未对其的内存访问会出错。
#define mp_align(n, alignment) (((n)(alignment-1)) ~(alignment-1))
#define mp_align_ptr(p, alignment) (void *)((((size_t)p)(alignment-1)) ~(alignment-1))
创建与销毁内存池
创建一个线程池其核心是创建struct mp_pool_s这个结构体并申请4k内存将各个指针指向上文初始状态的图一样。 销毁内存池遍历小块结构体和大块结构体进行free释放内存。
//创建内存池
struct mp_pool_s *mp_create_pool(size_t size) {struct mp_pool_s *pool;if (size PAGE_SIZE || size % PAGE_SIZE ! 0) {size PAGE_SIZE;}//分配4k以上不用malloc用posix_memalign/*int posix_memalign (void **memptr, size_t alignment, size_t size);*/int ret posix_memalign((void **) pool, MP_ALIGNMENT, size); //4K mp_pool_sif (ret) {return NULL;}pool-large NULL;pool-current pool-head (unsigned char *) pool sizeof(struct mp_pool_s);pool-head-last (unsigned char *) pool sizeof(struct mp_pool_s) sizeof(struct mp_node_s);pool-head-end (unsigned char *) pool PAGE_SIZE;pool-head-failed 0;return pool;
}//销毁内存池
void mp_destroy_pool(struct mp_pool_s *pool) {struct mp_large_s *large;for (large pool-large; large; large large-next) {if (large-alloc) {free(large-alloc);}}struct mp_node_s *cur, *next;cur pool-head-next;while (cur) {next cur-next;free(cur);cur next;}free(pool);
} 提供给用户的内存申请api
申请的内存以size做区分如果大于4k就分配大块内存小于4k就去block里面划分。
//分配内存
void *mp_malloc(struct mp_pool_s *pool, size_t size) {if (size 0) {return NULL;}if (size PAGE_SIZE - sizeof(struct mp_node_s)) {//largereturn mp_malloc_large(pool, size);}else {//smallunsigned char *mem_addr NULL;struct mp_node_s *cur NULL;cur pool-current;while (cur) {mem_addr mp_align_ptr(cur-last, MP_ALIGNMENT);if (cur-end - mem_addr size) {cur-quote;//引用1cur-last mem_addr size;return mem_addr;}else {cur cur-next;}}return mp_malloc_block(pool, size);// open new space}
}
void *mp_calloc(struct mp_pool_s *pool, size_t size) {void *mem_addr mp_malloc(pool, size);if (mem_addr) {memset(mem_addr, 0, size);}return mem_addr;
}
小块内存block扩容
所有的block都 e n d − l a s t s i z e end - last size end−lastsize 时进行扩容将新block的last后移即可。
//new block 4k
void *mp_malloc_block(struct mp_pool_s *pool, size_t size) {unsigned char *block;int ret posix_memalign((void **) block, MP_ALIGNMENT, PAGE_SIZE); //4Kif (ret) {return NULL;}struct mp_node_s *new_node (struct mp_node_s *) block;new_node-end block PAGE_SIZE;new_node-next NULL;unsigned char *ret_addr mp_align_ptr(block sizeof(struct mp_node_s), MP_ALIGNMENT);new_node-last ret_addr size;new_node-quote;struct mp_node_s *current pool-current;struct mp_node_s *cur NULL;for (cur current; cur-next; cur cur-next) {if (cur-failed 4) {current cur-next;}}//now cur last nodecur-next new_node;pool-current current;return ret_addr;
}
分配大块内存
//size4k
void *mp_malloc_large(struct mp_pool_s *pool, size_t size) {unsigned char *big_addr;int ret posix_memalign((void **) big_addr, MP_ALIGNMENT, size); //sizeif (ret) {return NULL;}struct mp_large_s *large;//released struct large resumeint n 0;for (large pool-large; large; large large-next) {if (large-alloc NULL) {large-size size;large-alloc big_addr;return big_addr;}if (n 3) {break;// 为了避免过多的遍历限制次数}}large mp_malloc(pool, sizeof(struct mp_large_s));if (large NULL) {free(big_addr);return NULL;}large-size size;large-alloc big_addr;large-next pool-large;pool-large large;return big_addr;
}
释放内存
如果是大块内存找到之后直接释放如果是小块内存将引用计数减1如果引用计数为0则重置last。
//释放内存
void mp_free(struct mp_pool_s *pool, void *p) {struct mp_large_s *large;for (large pool-large; large; large large-next) {//大块if (p large-alloc) {free(large-alloc);large-size 0;large-alloc NULL;return;}}//小块 引用-1struct mp_node_s *cur NULL;for (cur pool-head; cur; cur cur-next) {
// printf(cur:%p p:%p end:%p\n, (unsigned char *) cur, (unsigned char *) p, (unsigned char *) cur-end);if ((unsigned char *) cur (unsigned char *) p (unsigned char *) p (unsigned char *) cur-end) {cur-quote--;if (cur-quote 0) {if (cur pool-head) {pool-head-last (unsigned char *) pool sizeof(struct mp_pool_s) sizeof(struct mp_node_s);}else {cur-last (unsigned char *) cur sizeof(struct mp_node_s);}cur-failed 0;pool-current pool-head;}return;}}
}
内存池测试
//
// Created by 68725 on 2022/7/26.
//
#include stdlib.h
#include stdio.h
#include string.h#define PAGE_SIZE 4096
#define MP_ALIGNMENT 16
#define mp_align(n, alignment) (((n)(alignment-1)) ~(alignment-1))
#define mp_align_ptr(p, alignment) (void *)((((size_t)p)(alignment-1)) ~(alignment-1))//每4k一block结点
struct mp_node_s {unsigned char *end;//块的结尾unsigned char *last;//使用到哪了struct mp_node_s *next;//链表int quote;//引用计数int failed;//失效次数
};struct mp_large_s {struct mp_large_s *next;//链表int size;//alloc的大小void *alloc;//大块内存的起始地址
};struct mp_pool_s {struct mp_large_s *large;struct mp_node_s *head;struct mp_node_s *current;
};struct mp_pool_s *mp_create_pool(size_t size);void mp_destroy_pool(struct mp_pool_s *pool);void *mp_malloc(struct mp_pool_s *pool, size_t size);void *mp_calloc(struct mp_pool_s *pool, size_t size);void mp_free(struct mp_pool_s *pool, void *p);void mp_reset_pool(struct mp_pool_s *pool);void monitor_mp_poll(struct mp_pool_s *pool, char *tk);void mp_reset_pool(struct mp_pool_s *pool) {struct mp_node_s *cur;struct mp_large_s *large;for (large pool-large; large; large large-next) {if (large-alloc) {free(large-alloc);}}pool-large NULL;pool-current pool-head;for (cur pool-head; cur; cur cur-next) {cur-last (unsigned char *) cur sizeof(struct mp_node_s);cur-failed 0;cur-quote 0;}
}//创建内存池
struct mp_pool_s *mp_create_pool(size_t size) {struct mp_pool_s *pool;if (size PAGE_SIZE || size % PAGE_SIZE ! 0) {size PAGE_SIZE;}//分配4k以上不用malloc用posix_memalign/*int posix_memalign (void **memptr, size_t alignment, size_t size);*/int ret posix_memalign((void **) pool, MP_ALIGNMENT, size); //4K mp_pool_sif (ret) {return NULL;}pool-large NULL;pool-current pool-head (unsigned char *) pool sizeof(struct mp_pool_s);pool-head-last (unsigned char *) pool sizeof(struct mp_pool_s) sizeof(struct mp_node_s);pool-head-end (unsigned char *) pool PAGE_SIZE;pool-head-failed 0;return pool;
}//销毁内存池
void mp_destroy_pool(struct mp_pool_s *pool) {struct mp_large_s *large;for (large pool-large; large; large large-next) {if (large-alloc) {free(large-alloc);}}struct mp_node_s *cur, *next;cur pool-head-next;while (cur) {next cur-next;free(cur);cur next;}free(pool);
}//size4k
void *mp_malloc_large(struct mp_pool_s *pool, size_t size) {unsigned char *big_addr;int ret posix_memalign((void **) big_addr, MP_ALIGNMENT, size); //sizeif (ret) {return NULL;}struct mp_large_s *large;//released struct large resumeint n 0;for (large pool-large; large; large large-next) {if (large-alloc NULL) {large-size size;large-alloc big_addr;return big_addr;}if (n 3) {break;// 为了避免过多的遍历限制次数}}large mp_malloc(pool, sizeof(struct mp_large_s));if (large NULL) {free(big_addr);return NULL;}large-size size;large-alloc big_addr;large-next pool-large;pool-large large;return big_addr;
}//new block 4k
void *mp_malloc_block(struct mp_pool_s *pool, size_t size) {unsigned char *block;int ret posix_memalign((void **) block, MP_ALIGNMENT, PAGE_SIZE); //4Kif (ret) {return NULL;}struct mp_node_s *new_node (struct mp_node_s *) block;new_node-end block PAGE_SIZE;new_node-next NULL;unsigned char *ret_addr mp_align_ptr(block sizeof(struct mp_node_s), MP_ALIGNMENT);new_node-last ret_addr size;new_node-quote;struct mp_node_s *current pool-current;struct mp_node_s *cur NULL;for (cur current; cur-next; cur cur-next) {if (cur-failed 4) {current cur-next;}}//now cur last nodecur-next new_node;pool-current current;return ret_addr;
}//分配内存
void *mp_malloc(struct mp_pool_s *pool, size_t size) {if (size 0) {return NULL;}if (size PAGE_SIZE - sizeof(struct mp_node_s)) {//largereturn mp_malloc_large(pool, size);}else {//smallunsigned char *mem_addr NULL;struct mp_node_s *cur NULL;cur pool-current;while (cur) {mem_addr mp_align_ptr(cur-last, MP_ALIGNMENT);if (cur-end - mem_addr size) {cur-quote;//引用1cur-last mem_addr size;return mem_addr;}else {cur cur-next;}}return mp_malloc_block(pool, size);// open new space}
}void *mp_calloc(struct mp_pool_s *pool, size_t size) {void *mem_addr mp_malloc(pool, size);if (mem_addr) {memset(mem_addr, 0, size);}return mem_addr;
}//释放内存
void mp_free(struct mp_pool_s *pool, void *p) {struct mp_large_s *large;for (large pool-large; large; large large-next) {//大块if (p large-alloc) {free(large-alloc);large-size 0;large-alloc NULL;return;}}//小块 引用-1struct mp_node_s *cur NULL;for (cur pool-head; cur; cur cur-next) {
// printf(cur:%p p:%p end:%p\n, (unsigned char *) cur, (unsigned char *) p, (unsigned char *) cur-end);if ((unsigned char *) cur (unsigned char *) p (unsigned char *) p (unsigned char *) cur-end) {cur-quote--;if (cur-quote 0) {if (cur pool-head) {pool-head-last (unsigned char *) pool sizeof(struct mp_pool_s) sizeof(struct mp_node_s);}else {cur-last (unsigned char *) cur sizeof(struct mp_node_s);}cur-failed 0;pool-current pool-head;}return;}}
}void monitor_mp_poll(struct mp_pool_s *pool, char *tk) {printf(\r\n\r\n------start monitor poll------%s\r\n\r\n, tk);struct mp_node_s *head NULL;int i 0;for (head pool-head; head; head head-next) {i;if (pool-current head) {printf(current第%d块\n, i);}if (i 1) {printf(第%02d块small block 已使用:%4ld 剩余空间:%4ld 引用:%4d failed:%4d\n, i,(unsigned char *) head-last - (unsigned char *) pool,head-end - head-last, head-quote, head-failed);}else {printf(第%02d块small block 已使用:%4ld 剩余空间:%4ld 引用:%4d failed:%4d\n, i,(unsigned char *) head-last - (unsigned char *) head,head-end - head-last, head-quote, head-failed);}}struct mp_large_s *large;i 0;for (large pool-large; large; large large-next) {i;if (large-alloc ! NULL) {printf(第%d块large block size%d\n, i, large-size);}}printf(\r\n\r\n------stop monitor poll------\r\n\r\n);
}int main() {struct mp_pool_s *p mp_create_pool(PAGE_SIZE);monitor_mp_poll(p, create memory pool);
#if 0printf(mp_align(5, %d): %d, mp_align(17, %d): %d\n, MP_ALIGNMENT, mp_align(5, MP_ALIGNMENT), MP_ALIGNMENT,mp_align(17, MP_ALIGNMENT));printf(mp_align_ptr(p-current, %d): %p, p-current: %p\n, MP_ALIGNMENT, mp_align_ptr(p-current, MP_ALIGNMENT),p-current);
#endifvoid *mp[30];int i;for (i 0; i 30; i) {mp[i] mp_malloc(p, 512);}monitor_mp_poll(p, 申请512字节30个);for (i 0; i 30; i) {mp_free(p, mp[i]);}monitor_mp_poll(p, 销毁512字节30个);int j;for (i 0; i 50; i) {char *pp mp_calloc(p, 32);for (j 0; j 32; j) {if (pp[j]) {printf(calloc wrong\n);exit(-1);}}}monitor_mp_poll(p, 申请32字节50个);for (i 0; i 50; i) {char *pp mp_malloc(p, 3);}monitor_mp_poll(p, 申请3字节50个);void *pp[10];for (i 0; i 10; i) {pp[i] mp_malloc(p, 5120);}monitor_mp_poll(p, 申请大内存5120字节10个);for (i 0; i 10; i) {mp_free(p, pp[i]);}monitor_mp_poll(p, 销毁大内存5120字节10个);mp_reset_pool(p);monitor_mp_poll(p, reset pool);for (i 0; i 100; i) {void *s mp_malloc(p, 256);}monitor_mp_poll(p, 申请256字节100个);mp_destroy_pool(p);return 0;
} nginx内存池对比分析
相关结构体定义对比 创建内存池对比 内存申请对比
