When memco lru moved, mem_cgroup_move_lists() is called. thus, I add noreclaim lru logic at that point. Signed-off-by: KOSAKI Motohiro --- include/linux/memcontrol.h | 2 - mm/memcontrol.c | 73 +++++++++++++++++++++++++++------------------ mm/swap.c | 2 - mm/vmscan.c | 10 ++++-- 4 files changed, 54 insertions(+), 33 deletions(-) Index: b/mm/memcontrol.c =================================================================== --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -161,9 +161,10 @@ struct page_cgroup { int ref_cnt; /* cached, mapped, migrating */ int flags; }; -#define PAGE_CGROUP_FLAG_CACHE (0x1) /* charged as cache */ -#define PAGE_CGROUP_FLAG_ACTIVE (0x2) /* page is active in this cgroup */ -#define PAGE_CGROUP_FLAG_FILE (0x4) /* page is file system backed */ +#define PAGE_CGROUP_FLAG_CACHE (0x1) /* charged as cache */ +#define PAGE_CGROUP_FLAG_ACTIVE (0x2) /* page is active in this cgroup */ +#define PAGE_CGROUP_FLAG_FILE (0x4) /* page is file system backed */ +#define PAGE_CGROUP_FLAG_NORECLAIM (0x8) /* page is noreclaimable page */ static int page_cgroup_nid(struct page_cgroup *pc) { @@ -283,10 +284,14 @@ static void __mem_cgroup_remove_list(str { int lru = LRU_BASE; - if (pc->flags & PAGE_CGROUP_FLAG_ACTIVE) - lru += LRU_ACTIVE; - if (pc->flags & PAGE_CGROUP_FLAG_FILE) - lru += LRU_FILE; + if (pc->flags & PAGE_CGROUP_FLAG_NORECLAIM) + lru = LRU_NORECLAIM; + else { + if (pc->flags & PAGE_CGROUP_FLAG_ACTIVE) + lru += LRU_ACTIVE; + if (pc->flags & PAGE_CGROUP_FLAG_FILE) + lru += LRU_FILE; + } MEM_CGROUP_ZSTAT(mz, lru) -= 1; @@ -299,10 +304,14 @@ static void __mem_cgroup_add_list(struct { int lru = LRU_BASE; - if (pc->flags & PAGE_CGROUP_FLAG_ACTIVE) - lru += LRU_ACTIVE; - if (pc->flags & PAGE_CGROUP_FLAG_FILE) - lru += LRU_FILE; + if (pc->flags & PAGE_CGROUP_FLAG_NORECLAIM) + lru = LRU_NORECLAIM; + else { + if (pc->flags & PAGE_CGROUP_FLAG_ACTIVE) + lru += LRU_ACTIVE; + if (pc->flags & PAGE_CGROUP_FLAG_FILE) + lru += LRU_FILE; + } MEM_CGROUP_ZSTAT(mz, lru) += 1; list_add(&pc->lru, &mz->lists[lru]); @@ -310,21 +319,31 @@ static void __mem_cgroup_add_list(struct mem_cgroup_charge_statistics(pc->mem_cgroup, pc->flags, true); } -static void __mem_cgroup_move_lists(struct page_cgroup *pc, bool active) +static void __mem_cgroup_move_lists(struct page_cgroup *pc, enum lru_list lru) { struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc); - int from = pc->flags & PAGE_CGROUP_FLAG_ACTIVE; - int file = pc->flags & PAGE_CGROUP_FLAG_FILE; - int lru = LRU_FILE * !!file + !!from; + int active = pc->flags & PAGE_CGROUP_FLAG_ACTIVE; + int file = pc->flags & PAGE_CGROUP_FLAG_FILE; + int noreclaim = pc->flags & PAGE_CGROUP_FLAG_NORECLAIM; + enum lru_list from = noreclaim ? LRU_NORECLAIM : + (LRU_FILE * !!file + !!active); - MEM_CGROUP_ZSTAT(mz, lru) -= 1; + if (lru == from) + return; - if (active) - pc->flags |= PAGE_CGROUP_FLAG_ACTIVE; - else + MEM_CGROUP_ZSTAT(mz, from) -= 1; + + if (is_noreclaim_lru(lru)) { pc->flags &= ~PAGE_CGROUP_FLAG_ACTIVE; + pc->flags |= PAGE_CGROUP_FLAG_NORECLAIM; + } else { + if (is_active_lru(lru)) + pc->flags |= PAGE_CGROUP_FLAG_ACTIVE; + else + pc->flags &= ~PAGE_CGROUP_FLAG_ACTIVE; + pc->flags &= ~PAGE_CGROUP_FLAG_NORECLAIM; + } - lru = LRU_FILE * !!file + !!active; MEM_CGROUP_ZSTAT(mz, lru) += 1; list_move(&pc->lru, &mz->lists[lru]); } @@ -342,7 +361,7 @@ int task_in_mem_cgroup(struct task_struc /* * This routine assumes that the appropriate zone's lru lock is already held */ -void mem_cgroup_move_lists(struct page *page, bool active) +void mem_cgroup_move_lists(struct page *page, enum lru_list lru) { struct page_cgroup *pc; struct mem_cgroup_per_zone *mz; @@ -362,7 +381,7 @@ void mem_cgroup_move_lists(struct page * if (pc) { mz = page_cgroup_zoneinfo(pc); spin_lock_irqsave(&mz->lru_lock, flags); - __mem_cgroup_move_lists(pc, active); + __mem_cgroup_move_lists(pc, lru); spin_unlock_irqrestore(&mz->lru_lock, flags); } unlock_page_cgroup(page); @@ -460,12 +479,10 @@ unsigned long mem_cgroup_isolate_pages(u /* * TODO: play better with lumpy reclaim, grabbing anything. */ - if (PageActive(page) && !active) { - __mem_cgroup_move_lists(pc, true); - continue; - } - if (!PageActive(page) && active) { - __mem_cgroup_move_lists(pc, false); + if (PageNoreclaim(page) || + (PageActive(page) && !active) || + (!PageActive(page) && active)) { + __mem_cgroup_move_lists(pc, page_lru(page)); continue; } Index: b/mm/swap.c =================================================================== --- a/mm/swap.c +++ b/mm/swap.c @@ -189,7 +189,7 @@ void activate_page(struct page *page) lru += LRU_ACTIVE; add_page_to_lru_list(zone, page, lru); __count_vm_event(PGACTIVATE); - mem_cgroup_move_lists(page, true); + mem_cgroup_move_lists(page, lru); if (file) { zone->recent_scanned_file++; Index: b/mm/vmscan.c =================================================================== --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -475,13 +475,16 @@ int putback_lru_page(struct page *page) * non-reclaimable page on [in]active list. * We know how to handle that. */ - lru_cache_add_lru(page, lru + page_file_cache(page)); + lru += page_file_cache(page); + lru_cache_add_lru(page, lru); + mem_cgroup_move_lists(page, lru); } else { /* * Put non-reclaimable pages directly on zone's noreclaim * list. */ add_page_to_noreclaim_list(page); + mem_cgroup_move_lists(page, LRU_NORECLAIM); } put_page(page); /* drop ref from isolate */ @@ -1065,6 +1068,7 @@ static unsigned long shrink_inactive_lis } SetPageLRU(page); add_page_to_lru_list(zone, page, lru); + mem_cgroup_move_lists(page, lru); if (!pagevec_add(&pvec, page)) { spin_unlock_irq(&zone->lru_lock); __pagevec_release(&pvec); @@ -1202,7 +1206,7 @@ static void shrink_active_list(unsigned ClearPageActive(page); list_move(&page->lru, &zone->list[lru]); - mem_cgroup_move_lists(page, false); + mem_cgroup_move_lists(page, lru); pgmoved++; if (!pagevec_add(&pvec, page)) { __mod_zone_page_state(zone, NR_INACTIVE_ANON + lru, @@ -1234,7 +1238,7 @@ static void shrink_active_list(unsigned VM_BUG_ON(!PageActive(page)); list_move(&page->lru, &zone->list[lru]); - mem_cgroup_move_lists(page, true); + mem_cgroup_move_lists(page, lru); pgmoved++; if (!pagevec_add(&pvec, page)) { __mod_zone_page_state(zone, NR_INACTIVE_ANON + lru, Index: b/include/linux/memcontrol.h =================================================================== --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -35,7 +35,7 @@ extern int mem_cgroup_charge(struct page extern int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask); extern void mem_cgroup_uncharge_page(struct page *page); -extern void mem_cgroup_move_lists(struct page *page, bool active); +extern void mem_cgroup_move_lists(struct page *page, enum lru_list lru); extern unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan, struct list_head *dst, unsigned long *scanned, int order,